From 879271f5254259ab798d59a9ddf9067ba885ae01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Fri, 25 Aug 2023 16:34:39 +0200 Subject: [PATCH 01/33] Dummy change to test 2T6S pipeline on CI with rectified results data --- narps_open/pipelines/team_2T6S.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/narps_open/pipelines/team_2T6S.py b/narps_open/pipelines/team_2T6S.py index 42aa344f..0ec99d75 100755 --- a/narps_open/pipelines/team_2T6S.py +++ b/narps_open/pipelines/team_2T6S.py @@ -1,7 +1,7 @@ #!/usr/bin/python # coding: utf-8 -""" Write the work of NARPS' team 2T6S using Nipype """ +""" Write the work of NARPS team 2T6S using Nipype """ from os.path import join from itertools import product From b306fe469667e40ac71c28ac31f093cc58e79a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 29 Aug 2023 09:09:41 +0200 Subject: [PATCH 02/33] [T54A] refactoring --- .github/workflows/pipeline_tests.yml | 9 + narps_open/pipelines/__init__.py | 2 +- narps_open/pipelines/team_T54A.py | 1417 ++++++++++++++------------ tests/pipelines/test_team_T54A.py | 68 ++ 4 files changed, 870 insertions(+), 626 deletions(-) create mode 100644 tests/pipelines/test_team_T54A.py diff --git a/.github/workflows/pipeline_tests.yml b/.github/workflows/pipeline_tests.yml index f6cef4b4..a1d03b65 100644 --- a/.github/workflows/pipeline_tests.yml +++ b/.github/workflows/pipeline_tests.yml @@ -73,6 +73,15 @@ jobs: pytest -s -q -m "pipeline_test" ${{ needs.identify-tests.outputs.tests }} fi + - name: Archive pipeline execution failures + if: ${{ failure() }} # Run only if previous job fails + uses: actions/upload-artifact@v3 + with: + name: pipeline_tests-reports + path: | + crash-*.pklz + retention-days: 15 + - name: Report results on GitHub run: | # Start report diff --git a/narps_open/pipelines/__init__.py b/narps_open/pipelines/__init__.py index c3834fb6..8b62df29 100644 --- a/narps_open/pipelines/__init__.py +++ b/narps_open/pipelines/__init__.py @@ -68,7 +68,7 @@ 'R7D1': None, 'R9K3': None, 'SM54': None, - 'T54A': None, + 'T54A': 'PipelineTeamT54A', 'U26C': None, 'UI76': None, 'UK24': None, diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 1bb10dcd..b54a0ee7 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -1,630 +1,797 @@ -from nipype.interfaces.fsl import (BET, ICA_AROMA, FAST, MCFLIRT, FLIRT, FNIRT, ApplyWarp, SUSAN, - Info, ImageMaths, IsotropicSmooth, Threshold, Level1Design, FEATModel, - L2Model, Merge, FLAMEO, ContrastMgr,Cluster, FILMGLS, Randomise, MultipleRegressDesign) -from nipype.algorithms.modelgen import SpecifyModel +#!/usr/bin/python +# coding: utf-8 + +""" Write the work of NARPS team T54A using Nipype """ + +from os import system +from os.path import join +from itertools import product -from niflow.nipype1.workflows.fmri.fsl import create_susan_smooth +from nipype import Workflow, Node from nipype.interfaces.utility import IdentityInterface, Function from nipype.interfaces.io import SelectFiles, DataSink -from nipype.algorithms.misc import Gunzip -from nipype import Workflow, Node, MapNode -from nipype.interfaces.base import Bunch - -from os.path import join as opj -import os - -def get_session_infos(event_file): - ''' - Create Bunchs for specifyModel. - - Parameters : - - event_file : str, file corresponding to the run and the subject to analyze - - Returns : - - subject_info : list of Bunch for 1st level analysis. - ''' - from os.path import join as opj - from nipype.interfaces.base import Bunch - import numpy as np - - cond_names = ['trial', 'gain', 'loss', 'difficulty','response'] - - onset = {} - duration = {} - amplitude = {} - - for c in cond_names: # For each condition. - onset.update({c : []}) # creates dictionary items with empty lists - duration.update({c : []}) - amplitude.update({c : []}) - - with open(event_file, 'rt') as f: - next(f) # skip the header - - for line in f: - info = line.strip().split() - # Creates list with onsets, duration and loss/gain for amplitude (FSL) - for c in cond_names: - if info[5] != 'NoResp': - if c == 'gain': - onset[c].append(float(info[0])) - duration[c].append(float(info[4])) - amplitude[c].append(float(info[2])) - elif c == 'loss': - onset[c].append(float(info[0])) - duration[c].append(float(info[4])) - amplitude[c].append(float(info[3])) - elif c == 'trial': - onset[c].append(float(info[0])) - duration[c].append(float(info[4])) - amplitude[c].append(float(1)) - elif c == 'difficulty': - onset[c].append(float(info[0])) - duration[c].append(float(info[4])) - amplitude[c].append(abs(0.5 * float(info[2]) - float(info[3]))) - elif c == 'response': - onset[c].append(float(info[0]) + float(info[4])) - duration[c].append(float(0)) - amplitude[c].append(float(1)) - else: - if c=='missed': - onset[c].append(float(info[0])) - duration[c].append(float(0)) - - #for c in ['gain', 'loss']: - # amplitude[c] = amplitude[c] - np.mean(amplitude[c]) - - - subject_info = [] - - subject_info.append(Bunch(conditions=cond_names, - onsets=[onset[k] for k in cond_names], - durations=[duration[k] for k in cond_names], - amplitudes=[amplitude[k] for k in cond_names], - regressor_names=None, - regressors=None)) - - return subject_info - -def get_parameters_file(file, subject_id, run_id, result_dir, working_dir): - ''' - Create new tsv files with only desired parameters per subject per run. - - Parameters : - - filepaths : paths to subject parameters file (i.e. one per run) - - subject_id : subject for whom the 1st level analysis is made - - run_id: run for which the 1st level analysis is made - - result_dir: str, directory where results will be stored - - working_dir: str, name of the sub-directory for intermediate results - - Return : - - parameters_file : paths to new files containing only desired parameters. - ''' - import pandas as pd - import numpy as np - from os.path import join as opj - import os - - parameters_file = [] - - df = pd.read_csv(file, sep = '\t', header=0) - if 'NonSteadyStateOutlier00' in df.columns: - temp_list = np.array([df['X'], df['Y'], df['Z'], - df['RotX'], df['RotY'], df['RotZ'], - df['NonSteadyStateOutlier00']]) - else: - temp_list = np.array([df['X'], df['Y'], df['Z'], - df['RotX'], df['RotY'], df['RotZ']])# Parameters we want to use for the model - retained_parameters = pd.DataFrame(np.transpose(temp_list)) - new_path =opj(result_dir, working_dir, 'parameters_file', f"parameters_file_sub-{subject_id}_run{run_id}.tsv") - if not os.path.isdir(opj(result_dir, working_dir, 'parameters_file')): - os.mkdir(opj(result_dir, working_dir, 'parameters_file')) - writer = open(new_path, "w") - writer.write(retained_parameters.to_csv(sep = '\t', index = False, header = False, na_rep = '0.0')) - writer.close() - - parameters_file = new_path - os.system('export PATH=$PATH:/local/egermani/ICA-AROMA') - - return parameters_file - -# Linear contrast effects: 'Gain' vs. baseline, 'Loss' vs. baseline. -def get_contrasts(subject_id): - ''' - Create the list of tuples that represents contrasts. - Each contrast is in the form : - (Name,Stat,[list of condition names],[weights on those conditions]) - - Parameters: - - subject_id: str, ID of the subject - - Returns: - - contrasts: list of tuples, list of contrasts to analyze - ''' - # list of condition names - conditions = ['trial', 'gain', 'loss'] - - # create contrasts - gain = ('gain', 'T', conditions, [0, 1, 0]) - - loss = ('loss', 'T', conditions, [0, 0, 1]) - - # contrast list - contrasts = [gain, loss] - - return contrasts - - -def rm_smoothed_files(files, subject_id, run_id, result_dir, working_dir): - import shutil - from os.path import join as opj - - smooth_dir = opj(result_dir, working_dir, 'l1_analysis', f"_run_id_{run_id}_subject_id_{subject_id}", 'smooth') - - try: - shutil.rmtree(smooth_dir) - except OSError as e: - print(e) - else: - print("The directory is deleted successfully") - - -def get_l1_analysis(subject_list, run_list, TR, fwhm, exp_dir, output_dir, working_dir, result_dir): - """ - Returns the first level analysis workflow. - - Parameters: - - exp_dir: str, directory where raw data are stored - - result_dir: str, directory where results will be stored - - working_dir: str, name of the sub-directory for intermediate results - - output_dir: str, name of the sub-directory for final results - - subject_list: list of str, list of subject for which you want to do the analysis - - run_list: list of str, list of runs for which you want to do the analysis - - fwhm: float, fwhm for smoothing step - - TR: float, time repetition used during acquisition - - Returns: - - l1_analysis : Nipype WorkFlow - """ - # Infosource Node - To iterate on subject and runs - infosource = Node(IdentityInterface(fields = ['subject_id', 'run_id']), name = 'infosource') - infosource.iterables = [('subject_id', subject_list), - ('run_id', run_list)] - - # Templates to select files node - func_file = opj('derivatives', 'fmriprep', 'sub-{subject_id}', 'func', - 'sub-{subject_id}_task-MGT_run-{run_id}_bold_space-MNI152NLin2009cAsym_preproc.nii.gz') - - event_file = opj('sub-{subject_id}', 'func', 'sub-{subject_id}_task-MGT_run-{run_id}_events.tsv') - - param_file = opj('derivatives', 'fmriprep', 'sub-{subject_id}', 'func', - 'sub-{subject_id}_task-MGT_run-{run_id}_bold_confounds.tsv') - - template = {'func' : func_file, 'event' : event_file, 'param' : param_file} - - # SelectFiles node - to select necessary files - selectfiles = Node(SelectFiles(template, base_directory=exp_dir), name = 'selectfiles') - - # DataSink Node - store the wanted results in the wanted repository - datasink = Node(DataSink(base_directory=result_dir, container=output_dir), name='datasink') - - ## Skullstripping - skullstrip = Node(BET(frac = 0.1, functional = True, mask = True), name = 'skullstrip') - - ## Smoothing - smooth = Node(IsotropicSmooth(fwhm = 6), name = 'smooth') - - # Node contrasts to get contrasts - contrasts = Node(Function(function=get_contrasts, - input_names=['subject_id'], - output_names=['contrasts']), - name='contrasts') - - # Get Subject Info - get subject specific condition information - get_subject_infos = Node(Function(input_names=['event_file'], - output_names=['subject_info'], - function=get_session_infos), - name='get_subject_infos') - - specify_model = Node(SpecifyModel(high_pass_filter_cutoff = 100, - input_units = 'secs', - time_repetition = TR), name = 'specify_model') - - parameters = Node(Function(function=get_parameters_file, - input_names=['file', 'subject_id', 'run_id', 'result_dir', 'working_dir'], - output_names=['parameters_file']), - name='parameters') - - parameters.inputs.result_dir = result_dir - parameters.inputs.working_dir = working_dir - - # First temporal derivatives of the two regressors were also used, - # along with temporal filtering (60 s) of all the independent variable time-series. - # No motion parameter regressors used. - - l1_design = Node(Level1Design(bases = {'dgamma':{'derivs' : True}}, - interscan_interval = TR, - model_serial_correlations = True), name = 'l1_design') - - model_generation = Node(FEATModel(), name = 'model_generation') - - model_estimate = Node(FILMGLS(), name='model_estimate') - - remove_smooth = Node(Function(input_names = ['subject_id', 'run_id', 'files', 'result_dir', 'working_dir'], - function = rm_smoothed_files), name = 'remove_smooth') - - remove_smooth.inputs.result_dir = result_dir - remove_smooth.inputs.working_dir = working_dir - - # Create l1 analysis workflow and connect its nodes - l1_analysis = Workflow(base_dir = opj(result_dir, working_dir), name = "l1_analysis") - - l1_analysis.connect([(infosource, selectfiles, [('subject_id', 'subject_id'), - ('run_id', 'run_id')]), - (selectfiles, get_subject_infos, [('event', 'event_file')]), - (selectfiles, parameters, [('param', 'file')]), - (infosource, contrasts, [('subject_id', 'subject_id')]), - (infosource, parameters, [('subject_id', 'subject_id'), - ('run_id', 'run_id')]), - (selectfiles, skullstrip, [('func', 'in_file')]), - (skullstrip, smooth, [('out_file', 'in_file')]), - (parameters, specify_model, [('parameters_file', 'realignment_parameters')]), - (smooth, specify_model, [('out_file', 'functional_runs')]), - (get_subject_infos, specify_model, [('subject_info', 'subject_info')]), - (contrasts, l1_design, [('contrasts', 'contrasts')]), - (specify_model, l1_design, [('session_info', 'session_info')]), - (l1_design, model_generation, [('ev_files', 'ev_files'), ('fsf_files', 'fsf_file')]), - (smooth, model_estimate, [('out_file', 'in_file')]), - (model_generation, model_estimate, [('con_file', 'tcon_file'), - ('design_file', 'design_file')]), - (infosource, remove_smooth, [('subject_id', 'subject_id'), ('run_id', 'run_id')]), - (model_estimate, remove_smooth, [('results_dir', 'files')]), - (model_estimate, datasink, [('results_dir', 'l1_analysis.@results')]), - (model_generation, datasink, [('design_file', 'l1_analysis.@design_file'), - ('design_image', 'l1_analysis.@design_img')]), - (skullstrip, datasink, [('mask_file', 'l1_analysis.@skullstriped')]) - ]) - - return l1_analysis - -def get_l2_analysis(subject_list, contrast_list, run_list, exp_dir, output_dir, working_dir, result_dir, data_dir): - """ - Returns the 2nd level of analysis workflow. - - Parameters: - - exp_dir: str, directory where raw data are stored - - result_dir: str, directory where results will be stored - - working_dir: str, name of the sub-directory for intermediate results - - output_dir: str, name of the sub-directory for final results - - subject_list: list of str, list of subject for which you want to do the preprocessing - - contrast_list: list of str, list of contrasts to analyze - - run_list: list of str, list of runs for which you want to do the analysis - - - Returns: - - l2_analysis: Nipype WorkFlow - """ - # Infosource Node - To iterate on subject and runs - infosource_2ndlevel = Node(IdentityInterface(fields=['subject_id', 'contrast_id']), name='infosource_2ndlevel') - infosource_2ndlevel.iterables = [('subject_id', subject_list), ('contrast_id', contrast_list)] - - # Templates to select files node - copes_file = opj(output_dir, 'l1_analysis', '_run_id_*_subject_id_{subject_id}', 'results', - 'cope{contrast_id}.nii.gz') - - varcopes_file = opj(output_dir, 'l1_analysis', '_run_id_*_subject_id_{subject_id}', 'results', - 'varcope{contrast_id}.nii.gz') - - mask_file = opj(data_dir, 'NARPS-T54A', 'hypo1_cope.nii.gz') - - template = {'cope' : copes_file, 'varcope' : varcopes_file, 'mask':mask_file} - - # SelectFiles node - to select necessary files - selectfiles_2ndlevel = Node(SelectFiles(template, base_directory=result_dir), name = 'selectfiles_2ndlevel') - - datasink_2ndlevel = Node(DataSink(base_directory=result_dir, container=output_dir), name='datasink_2ndlevel') - - # Generate design matrix - specify_model_2ndlevel = Node(L2Model(num_copes = len(run_list)), name='l2model_2ndlevel') - - # Merge copes and varcopes files for each subject - merge_copes_2ndlevel = Node(Merge(dimension='t'), name='merge_copes_2ndlevel') - - merge_varcopes_2ndlevel = Node(Merge(dimension='t'), name='merge_varcopes_2ndlevel') - - # Second level (single-subject, mean of all four scans) analyses: Fixed effects analysis. - flame = Node(FLAMEO(run_mode = 'flame1'), - name='flameo') - - l2_analysis = Workflow(base_dir = opj(result_dir, working_dir), name = "l2_analysis") - - l2_analysis.connect([(infosource_2ndlevel, selectfiles_2ndlevel, [('subject_id', 'subject_id'), - ('contrast_id', 'contrast_id')]), - (selectfiles_2ndlevel, merge_copes_2ndlevel, [('cope', 'in_files')]), - (selectfiles_2ndlevel, merge_varcopes_2ndlevel, [('varcope', 'in_files')]), - (selectfiles_2ndlevel, flame, [('mask', 'mask_file')]), - (merge_copes_2ndlevel, flame, [('merged_file', 'cope_file')]), - (merge_varcopes_2ndlevel, flame, [('merged_file', 'var_cope_file')]), - (specify_model_2ndlevel, flame, [('design_mat', 'design_file'), - ('design_con', 't_con_file'), - ('design_grp', 'cov_split_file')]), - (flame, datasink_2ndlevel, [('zstats', 'l2_analysis.@stats'), - ('tstats', 'l2_analysis.@tstats'), - ('copes', 'l2_analysis.@copes'), - ('var_copes', 'l2_analysis.@varcopes')])]) - - return l2_analysis - -def get_subgroups_contrasts(copes, varcopes, subject_ids, participants_file): - ''' - Parameters : - - copes: original file list selected by selectfiles node - - varcopes: original file list selected by selectfiles node - - subject_ids: list of subject IDs that are analyzed - - participants_file: str, file containing participants characteristics - - This function return the file list containing only the files belonging to subject in the wanted group. - ''' - - from os.path import join as opj - - equalRange_id = [] - equalIndifference_id = [] - - subject_list = ['sub-' + str(i) for i in subject_ids] - - with open(participants_file, 'rt') as f: - next(f) # skip the header - - for line in f: +from nipype.interfaces.fsl import ( + BET, IsotropicSmooth, Level1Design, FEATModel, L2Model, Merge, FLAMEO, + FILMGLS, Randomise, MultipleRegressDesign + ) +from nipype.algorithms.modelgen import SpecifyModel + +from narps_open.pipelines import Pipeline + +class PipelineTeamT54A(Pipeline): + """ A class that defines the pipeline of team T54A. """ + + def __init__(self): + super().__init__() + self.fwhm = 4.0 + self.team_id = 'T54A' + self.contrast_list = ['01', '02'] + + def get_preprocessing(self): + """ No preprocessing has been done by team T54A """ + return None + + def get_session_infos(event_file): + """ + Create Bunchs for specifyModel. + + Parameters : + - event_file : str, file corresponding to the run and the subject to analyze + + Returns : + - subject_info : list of Bunch for 1st level analysis. + """ + from nipype.interfaces.base import Bunch + + condition_names = ['trial', 'gain', 'loss', 'difficulty', 'response'] + onset = {} + duration = {} + amplitude = {} + + for condition in condition_names: + # Create dictionary items with empty lists + onset.update({condition : []}) + duration.update({condition : []}) + amplitude.update({condition : []}) + + with open(event_file, 'rt') as file: + next(file) # skip the header + + for line in file: info = line.strip().split() - - if info[0] in subject_list and info[1] == "equalIndifference": - equalIndifference_id.append(info[0][-3:]) - elif info[0] in subject_list and info[1] == "equalRange": - equalRange_id.append(info[0][-3:]) - - copes_equalIndifference = [] - copes_equalRange = [] - copes_global = [] - varcopes_equalIndifference = [] - varcopes_equalRange = [] - varcopes_global = [] - - for file in copes: - sub_id = file.split('/') - if sub_id[-1][4:7] in equalIndifference_id: - copes_equalIndifference.append(file) - elif sub_id[-1][4:7] in equalRange_id: - copes_equalRange.append(file) - if sub_id[-1][4:7] in subject_ids: - copes_global.append(file) - - for file in varcopes: - sub_id = file.split('/') - if sub_id[-1][4:7] in equalIndifference_id: - varcopes_equalIndifference.append(file) - elif sub_id[-1][4:7] in equalRange_id: - varcopes_equalRange.append(file) - if sub_id[-1][4:7] in subject_ids: - varcopes_global.append(file) - - return copes_equalIndifference, copes_equalRange, varcopes_equalIndifference, varcopes_equalRange, equalIndifference_id, equalRange_id, copes_global, varcopes_global - -def get_regs(equalRange_id, equalIndifference_id, method, subject_list): - """ - Create dictionary of regressors for group analysis. - - Parameters: - - equalRange_id: list of str, ids of subjects in equal range group - - equalIndifference_id: list of str, ids of subjects in equal indifference group - - method: one of "equalRange", "equalIndifference" or "groupComp" - - subject_list: list of str, ids of subject for which to do the analysis - - Returns: - - regressors: dict, dictionary of regressors used to distinguish groups in FSL group analysis - """ - if method == "equalRange": - regressors = dict(group_mean = [1 for i in range(len(equalRange_id))]) - - elif method == "equalIndifference": - regressors = dict(group_mean = [1 for i in range(len(equalIndifference_id))]) - - elif method == "groupComp": - equalRange_reg = [1 for i in range(len(equalRange_id) + len(equalIndifference_id))] - equalIndifference_reg = [0 for i in range(len(equalRange_id) + len(equalIndifference_id))] - - for i, sub_id in enumerate(subject_list): - if sub_id in equalIndifference_id: - index = i - equalIndifference_reg[index] = 1 - equalRange_reg[index] = 0 - - regressors = dict(equalRange = equalRange_reg, - equalIndifference = equalIndifference_reg) - - return regressors - -def get_group_workflow(subject_list, n_sub, contrast_list, method, exp_dir, output_dir, - working_dir, result_dir, data_dir): - """ - Returns the group level of analysis workflow. - - Parameters: - - exp_dir: str, directory where raw data are stored - - result_dir: str, directory where results will be stored - - working_dir: str, name of the sub-directory for intermediate results - - output_dir: str, name of the sub-directory for final results - - subject_list: list of str, list of subject for which you want to do the preprocessing - - contrast_list: list of str, list of contrasts to analyze - - method: one of "equalRange", "equalIndifference" or "groupComp" - - n_sub: int, number of subjects - - Returns: - - l2_analysis: Nipype WorkFlow - """ - # Infosource Node - To iterate on subject and runs - infosource_3rdlevel = Node(IdentityInterface(fields = ['contrast_id', 'exp_dir', 'result_dir', - 'output_dir', 'working_dir', 'subject_list', 'method'], - exp_dir = exp_dir, result_dir = result_dir, - output_dir = output_dir, working_dir = working_dir, - subject_list = subject_list, method = method), - name = 'infosource_3rdlevel') - infosource_3rdlevel.iterables = [('contrast_id', contrast_list)] - - # Templates to select files node - copes_file = opj(data_dir, 'NARPS-T54A', 'sub-*_contrast-{contrast_id}_cope.nii.gz') # contrast_id = ploss ou pgain - - varcopes_file = opj(data_dir, 'NARPS-T54A', 'sub-*_contrast-{contrast_id}_varcope.nii.gz') - - participants_file = opj(exp_dir, 'participants.tsv') - - mask_file = opj(data_dir, 'NARPS-T54A', 'hypo2_unthresh_Z.nii.gz') - - template = {'cope' : copes_file, 'varcope' : varcopes_file, 'participants' : participants_file, - 'mask' : mask_file} - - # SelectFiles node - to select necessary files - selectfiles_3rdlevel = Node(SelectFiles(template, base_directory=result_dir), name = 'selectfiles_3rdlevel') - - datasink_3rdlevel = Node(DataSink(base_directory=result_dir, container=output_dir), name='datasink_3rdlevel') - - merge_copes_3rdlevel = Node(Merge(dimension = 't'), name = 'merge_copes_3rdlevel') - merge_varcopes_3rdlevel = Node(Merge(dimension = 't'), name = 'merge_varcopes_3rdlevel') - - subgroups_contrasts = Node(Function(input_names = ['copes', 'varcopes', 'subject_ids', 'participants_file'], - output_names = ['copes_equalIndifference', 'copes_equalRange', - 'varcopes_equalIndifference', 'varcopes_equalRange', - 'equalIndifference_id', 'equalRange_id', 'copes_global', - 'varcopes_global'], - function = get_subgroups_contrasts), - name = 'subgroups_contrasts') - - specifymodel_3rdlevel = Node(MultipleRegressDesign(), name = 'specifymodel_3rdlevel') - - flame_3rdlevel = Node(FLAMEO(run_mode = 'flame1'), - name='flame_3rdlevel') - - regs = Node(Function(input_names = ['equalRange_id', 'equalIndifference_id', 'method', 'subject_list'], - output_names = ['regressors'], - function = get_regs), name = 'regs') - regs.inputs.method = method - regs.inputs.subject_list = subject_list - - randomise = Node(Randomise(num_perm = 10000, tfce=True, vox_p_values=True, c_thresh=0.05, tfce_E=0.01), - name = "randomise") - - l3_analysis = Workflow(base_dir = opj(result_dir, working_dir), name = f"l3_analysis_{method}_nsub_{n_sub}") - - l3_analysis.connect([(infosource_3rdlevel, selectfiles_3rdlevel, [('contrast_id', 'contrast_id')]), - (infosource_3rdlevel, subgroups_contrasts, [('subject_list', 'subject_ids')]), - (selectfiles_3rdlevel, subgroups_contrasts, [('cope', 'copes'), ('varcope', 'varcopes'), - ('participants', 'participants_file')]), - (selectfiles_3rdlevel, flame_3rdlevel, [('mask', 'mask_file')]), - (selectfiles_3rdlevel, randomise, [('mask', 'mask')]), - (subgroups_contrasts, regs, [('equalRange_id', 'equalRange_id'), - ('equalIndifference_id', 'equalIndifference_id')]), - (regs, specifymodel_3rdlevel, [('regressors', 'regressors')])]) - - - if method == 'equalIndifference' or method == 'equalRange': - specifymodel_3rdlevel.inputs.contrasts = [["group_mean", "T", ["group_mean"], [1]], ["group_mean_neg", "T", ["group_mean"], [-1]]] - - if method == 'equalIndifference': - l3_analysis.connect([(subgroups_contrasts, merge_copes_3rdlevel, - [('copes_equalIndifference', 'in_files')]), - (subgroups_contrasts, merge_varcopes_3rdlevel, - [('varcopes_equalIndifference', 'in_files')])]) - elif method == 'equalRange': - l3_analysis.connect([(subgroups_contrasts, merge_copes_3rdlevel, [('copes_equalRange', 'in_files')]), - (subgroups_contrasts, merge_varcopes_3rdlevel, [('varcopes_equalRange', 'in_files')])]) - - elif method == "groupComp": - specifymodel_3rdlevel.inputs.contrasts = [["equalRange_sup", "T", ["equalRange", "equalIndifference"], - [1, -1]]] - - l3_analysis.connect([(subgroups_contrasts, merge_copes_3rdlevel, [('copes_global', 'in_files')]), - (subgroups_contrasts, merge_varcopes_3rdlevel, [('varcopes_global', 'in_files')])]) - - l3_analysis.connect([(merge_copes_3rdlevel, flame_3rdlevel, [('merged_file', 'cope_file')]), - (merge_varcopes_3rdlevel, flame_3rdlevel, [('merged_file', 'var_cope_file')]), - (specifymodel_3rdlevel, flame_3rdlevel, [('design_mat', 'design_file'), - ('design_con', 't_con_file'), - ('design_grp', 'cov_split_file')]), - (merge_copes_3rdlevel, randomise, [('merged_file', 'in_file')]), - (specifymodel_3rdlevel, randomise, [('design_mat', 'design_mat'), - ('design_con', 'tcon')]), - (randomise, datasink_3rdlevel, [('t_corrected_p_files', - f"l3_analysis_{method}_nsub_{n_sub}.@tcorpfile"), - ('tstat_files', f"l3_analysis_{method}_nsub_{n_sub}.@tstat")]), - (flame_3rdlevel, datasink_3rdlevel, [('zstats', - f"l3_analysis_{method}_nsub_{n_sub}.@zstats"), - ('tstats', - f"l3_analysis_{method}_nsub_{n_sub}.@tstats")]), + # Creates list with onsets, duration and loss/gain + # for amplitude (FSL) + for condition in condition_names: + if info[5] != 'NoResp': + if condition == 'gain': + onset[condition].append(float(info[0])) + duration[condition].append(float(info[4])) + amplitude[condition].append(float(info[2])) + elif condition == 'loss': + onset[condition].append(float(info[0])) + duration[condition].append(float(info[4])) + amplitude[condition].append(float(info[3])) + elif condition == 'trial': + onset[condition].append(float(info[0])) + duration[condition].append(float(info[4])) + amplitude[condition].append(float(1)) + elif condition == 'difficonditionulty': + onset[condition].append(float(info[0])) + duration[condition].append(float(info[4])) + amplitude[condition].append( + abs(0.5 * float(info[2]) - float(info[3])) + ) + elif condition == 'response': + onset[condition].append(float(info[0]) + float(info[4])) + duration[condition].append(float(0)) + amplitude[condition].append(float(1)) + else: + if condition=='missed': + onset[condition].append(float(info[0])) + duration[condition].append(float(0)) + + return [ + Bunch( + conditions = condition_names, + onsets = [onset[k] for k in condition_names], + durations = [duration[k] for k in condition_names], + amplitudes = [amplitude[k] for k in condition_names], + regressor_names = None, + regressors = None) + ] + + def get_parameters_file(filepath, subject_id, run_id, working_dir): + """ + Create a tsv file with only desired parameters per subject per run. + + Parameters : + - filepath : path to subject parameters file (i.e. one per run) + - subject_id : subject for whom the 1st level analysis is made + - run_id: run for which the 1st level analysis is made + - working_dir: str, name of the directory for intermediate results + + Return : + - parameters_file : paths to new files containing only desired parameters. + """ + from os import mkdir + from os.path import join, isdir + + from pandas import read_csv, DataFrame + from numpy import array, transpose + + data_frame = read_csv(filepath, sep = '\t', header=0) + if 'NonSteadyStateOutlier00' in data_frame.columns: + temp_list = array([ + data_frame['X'], data_frame['Y'], data_frame['Z'], + data_frame['RotX'], data_frame['RotY'], data_frame['RotZ'], + data_frame['NonSteadyStateOutlier00']]) + else: + temp_list = array([ + data_frame['X'], data_frame['Y'], data_frame['Z'], + data_frame['RotX'], data_frame['RotY'], data_frame['RotZ']]) + retained_parameters = DataFrame(transpose(temp_list)) + + parameters_file = join(working_dir, 'parameters_file', + f'parameters_file_sub-{subject_id}_run{run_id}.tsv') + + if not isdir(join(working_dir, 'parameters_file')): + mkdir(join(working_dir, 'parameters_file')) + + with open(parameters_file, 'w') as writer: + writer.write(retained_parameters.to_csv( + sep = '\t', index = False, header = False, na_rep = '0.0')) + + return parameters_file + + def get_contrasts(): + """ + Create a list of tuples that represent contrasts. + Each contrast is in the form : + (Name,Stat,[list of condition names],[weights on those conditions]) + + Returns: + - contrasts: list of tuples, list of contrasts to analyze + """ + # List of condition names + conditions = ['trial', 'gain', 'loss'] + + # Create contrasts + gain = ('gain', 'T', conditions, [0, 1, 0]) + loss = ('loss', 'T', conditions, [0, 0, 1]) + + # Contrast list + return [gain, loss] + + def remove_smoothed_files(_, subject_id, run_id, working_dir): + """ + This method is used in a Function node to fully remove + the files generated by the smoothing node, once they aren't needed anymore. + + Parameters: + - _: Node input only used for triggering the Node + - subject_id: str, id of the subject from which to remove the file + - run_id: str, id of the run from which to remove the file + - working_dir: str, path to the working dir + """ + from shutil import rmtree + from os.path import join + + try: + rmtree(join( + working_dir, 'l1_analysis', + f'_run_id_{run_id}_subject_id_{subject_id}', 'smooth') + ) + except OSError as error: + print(error) + else: + print('The directory is deleted successfully') + + def get_run_level_analysis(self): + """ + Create the run level analysis workflow. + + Returns: + - l1_analysis : nipype.WorkFlow + """ + # Infosource Node - To iterate on subject and runs + infosource = Node(IdentityInterface( + fields = ['subject_id', 'run_id']), + name = 'infosource') + infosource.iterables = [ + ('subject_id', self.subject_list), + ('run_id', self.run_list)] + + # Templates to select files node + template = { + # Parameter file + 'param' : join('derivatives', 'fmriprep', 'sub-{subject_id}', 'func', + 'sub-{subject_id}_task-MGT_run-{run_id}_bold_confounds.tsv'), + # Functional MRI + 'func' : join('derivatives', 'fmriprep', 'sub-{subject_id}', 'func', + 'sub-{subject_id}_task-MGT_run-{run_id}_bold_space-MNI152NLin2009cAsym_preproc.nii.gz' + ), + # Event file + 'event' : join('sub-{subject_id}', 'func', + 'sub-{subject_id}_task-MGT_run-{run_id}_events.tsv') + } + + # SelectFiles - to select necessary files + selectfiles = Node(SelectFiles(template, base_directory = self.directories.dataset_dir), + name = 'selectfiles') + + # DataSink - store the wanted results in the wanted repository + datasink = Node(DataSink(base_directory = self.directories.output_dir), + name='datasink') + + # Skullstripping + skullstrip = Node(BET(frac = 0.1, functional = True, mask = True), + name = 'skullstrip') + + # Smoothing + smooth = Node(IsotropicSmooth(fwhm = self.fwhm), + name = 'smooth') + + # Function node get_subject_infos - get subject specific condition information + subject_infos = Node(Function( + function = self.get_session_infos, + input_names = ['event_file'], + output_names = ['subject_info']), + name = 'subject_infos') + + # SpecifyModel - generates Model + specify_model = Node(SpecifyModel( + high_pass_filter_cutoff = 100, input_units = 'secs', time_repetition = self.tr), + name = 'specify_model') + + # Node contrasts to get contrasts + contrasts = Node(Function( + function = self.get_contrasts, + input_names = ['subject_id'], + output_names = ['contrasts']), + name = 'contrasts') + + # Function node get_parameters_file - get parameters files + parameters = Node(Function( + function = self.get_parameters_file, + input_names = ['filepath', 'subject_id', 'run_id', 'working_dir'], + output_names=['parameters_file']), + name='parameters') + parameters.inputs.working_dir = self.directories.working_dir + + # First temporal derivatives of the two regressors were also used, + # along with temporal filtering (60 s) of all the independent variable time-series. + # No motion parameter regressors used. + # Level1Design - generates a design matrix + l1_design = Node(Level1Design( + bases = {'dgamma':{'derivs' : True}}, + interscan_interval = self.tr, + model_serial_correlations = True), + name = 'l1_design') + + model_generation = Node(FEATModel(), + name = 'model_generation') + + model_estimate = Node(FILMGLS(), + name='model_estimate') + + # Function node remove_smoothed_files - remove output of the smooth node + remove_smoothed_files = Node(Function( + function = self.remove_smoothed_files, + input_names = ['_', 'subject_id', 'run_id', 'working_dir']), + name = 'remove_smoothed_files') + remove_smoothed_files.inputs.working_dir = self.directories.working_dir + + # Create l1 analysis workflow and connect its nodes + l1_analysis = Workflow(base_dir = self.directories.working_dir, name = 'l1_analysis') + l1_analysis.connect([ + (infosource, selectfiles, [ + ('subject_id', 'subject_id'), + ('run_id', 'run_id')]), + (selectfiles, subject_infos, [('event', 'event_file')]), + (selectfiles, parameters, [('param', 'filepath')]), + (infosource, contrasts, [('subject_id', 'subject_id')]), + (infosource, parameters, [ + ('subject_id', 'subject_id'), + ('run_id', 'run_id')]), + (selectfiles, skullstrip, [('func', 'in_file')]), + (skullstrip, smooth, [('out_file', 'in_file')]), + (parameters, specify_model, [('parameters_file', 'realignment_parameters')]), + (smooth, specify_model, [('out_file', 'functional_runs')]), + (subject_infos, specify_model, [('subject_info', 'subject_info')]), + (contrasts, l1_design, [('contrasts', 'contrasts')]), + (specify_model, l1_design, [('session_info', 'session_info')]), + (l1_design, model_generation, [ + ('ev_files', 'ev_files'), + ('fsf_files', 'fsf_file')]), + (smooth, model_estimate, [('out_file', 'in_file')]), + (model_generation, model_estimate, [ + ('con_file', 'tcon_file'), + ('design_file', 'design_file')]), + (infosource, remove_smoothed_files, [ + ('subject_id', 'subject_id'), + ('run_id', 'run_id')]), + (model_estimate, remove_smoothed_files, [('results_dir', '_')]), + (model_estimate, datasink, [('results_dir', 'l1_analysis.@results')]), + (model_generation, datasink, [ + ('design_file', 'l1_analysis.@design_file'), + ('design_image', 'l1_analysis.@design_img')]), + (skullstrip, datasink, [('mask_file', 'l1_analysis.@skullstriped')]) ]) - - return l3_analysis - -def reorganize_results(result_dir, output_dir, n_sub, team_ID): - """ - Reorganize the results to analyze them. - - Parameters: - - result_dir: str, directory where results will be stored - - output_dir: str, name of the sub-directory for final results - - n_sub: float, number of subject used for the analysis - - team_ID: str, ID of the team to reorganize results - - """ - from os.path import join as opj - import os - import shutil - import gzip - import nibabel as nib - import numpy as np - - h1 = opj(result_dir, output_dir, f"l3_analysis_equalIndifference_nsub_{n_sub}", '_contrast_id_pgain') - h2 = opj(result_dir, output_dir, f"l3_analysis_equalRange_nsub_{n_sub}", '_contrast_id_pgain') - h3 = opj(result_dir, output_dir, f"l3_analysis_equalIndifference_nsub_{n_sub}", '_contrast_id_pgain') - h4 = opj(result_dir, output_dir, f"l3_analysis_equalRange_nsub_{n_sub}", '_contrast_id_pgain') - h5 = opj(result_dir, output_dir, f"l3_analysis_equalIndifference_nsub_{n_sub}", '_contrast_id_ploss') - h6 = opj(result_dir, output_dir, f"l3_analysis_equalRange_nsub_{n_sub}", '_contrast_id_ploss') - h7 = opj(result_dir, output_dir, f"l3_analysis_equalIndifference_nsub_{n_sub}", '_contrast_id_ploss') - h8 = opj(result_dir, output_dir, f"l3_analysis_equalRange_nsub_{n_sub}", '_contrast_id_ploss') - h9 = opj(result_dir, output_dir, f"l3_analysis_groupComp_nsub_{n_sub}", '_contrast_id_ploss') - - h = [h1, h2, h3, h4, h5, h6, h7, h8, h9] - - repro_unthresh = [opj(filename, "zstat2.nii.gz") if i in [4, 5] else opj(filename, "zstat1.nii.gz") for i, filename in enumerate(h)] - - repro_thresh = [opj(filename, "randomise_tfce_corrp_tstat2.nii.gz") if i in [4, 5] else opj(filename, 'randomise_tfce_corrp_tstat1.nii.gz') for i, filename in enumerate(h)] - - if not os.path.isdir(opj(result_dir, "NARPS-reproduction")): - os.mkdir(opj(result_dir, "NARPS-reproduction")) - - for i, filename in enumerate(repro_unthresh): - f_in = filename - f_out = opj(result_dir, "NARPS-reproduction", f"team_{team_ID}_nsub_{n_sub}_hypo{i+1}_unthresholded.nii.gz") - shutil.copyfile(f_in, f_out) - - for i, filename in enumerate(repro_thresh): - f_in = filename - f_out = opj(result_dir, "NARPS-reproduction", f"team_{team_ID}_nsub_{n_sub}_hypo{i+1}_thresholded.nii.gz") - shutil.copyfile(f_in, f_out) - - for i, filename in enumerate(repro_thresh): - f_in = filename - img = nib.load(filename) - original_affine = img.affine.copy() - spm = nib.load(repro_unthresh[i]) - new_img = img.get_fdata().astype(float) > 0 - new_img = new_img.astype(float) - print(np.unique(new_img)) - print(np.unique(spm.get_fdata())) - new_img = new_img * spm.get_fdata() - print(np.unique(new_img)) - new_spm = nib.Nifti1Image(new_img, original_affine) - nib.save(new_spm, opj(result_dir, "NARPS-reproduction", - f"team_{team_ID}_nsub_{n_sub}_hypo{i+1}_thresholded.nii.gz")) - - print(f"Results files of team {team_ID} reorganized.") + + return l1_analysis + + def get_run_level_outputs(self): + """ Return the names of the files the run level analysis is supposed to generate. """ + + parameters = { + 'run_id' : self.run_list, + 'subject_id' : self.subject_list, + 'file' : [ + join('results', 'cope1.nii.gz'), + join('results', 'cope2.nii.gz'), + join('results', 'dof'), + join('results', 'logfile'), + join('results', 'pe10.nii.gz'), + join('results', 'pe11.nii.gz'), + join('results', 'pe12.nii.gz'), + join('results', 'pe13.nii.gz'), + join('results', 'pe14.nii.gz'), + join('results', 'pe15.nii.gz'), + join('results', 'pe16.nii.gz'), + join('results', 'pe17.nii.gz'), + join('results', 'pe1.nii.gz'), + join('results', 'pe2.nii.gz'), + join('results', 'pe3.nii.gz'), + join('results', 'pe4.nii.gz'), + join('results', 'pe5.nii.gz'), + join('results', 'pe6.nii.gz'), + join('results', 'pe7.nii.gz'), + join('results', 'pe8.nii.gz'), + join('results', 'pe9.nii.gz'), + join('results', 'res4d.nii.gz'), + join('results', 'sigmasquareds.nii.gz'), + join('results', 'threshac1.nii.gz'), + join('results', 'tstat1.nii.gz'), + join('results', 'tstat2.nii.gz'), + join('results', 'varcope1.nii.gz'), + join('results', 'varcope2.nii.gz'), + join('results', 'zstat1.nii.gz'), + join('results', 'zstat2.nii.gz'), + 'run0.mat', + 'run0.png', + 'sub-{subject_id}_task-MGT_run-{run_id}_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz' + ] + } + parameter_sets = product(*parameters.values()) + template = join( + self.directories.output_dir, + 'l1_analysis', '_run_id_{run_id}_subject_id_{subject_id}','{file}' + ) + + return [template.format(**dict(zip(parameters.keys(), parameter_values)))\ + for parameter_values in parameter_sets] + + def get_subject_level_analysis(self): + """ + Create the subject level analysis workflow. + + Returns: + - l2_analysis: nipype.WorkFlow + """ + # Infosource Node - To iterate on subject and runs + infosource_sub_level = Node(IdentityInterface( + fields = ['subject_id', 'contrast_id']), + name = 'infosource_sub_level') + infosource_sub_level.iterables = [ + ('subject_id', self.subject_list), + ('contrast_id', self.contrast_list) + ] + + # Templates to select files node + templates = { + 'cope' : join(self.directories.output_dir, + 'l1_analysis', '_run_id_*_subject_id_{subject_id}', 'results', + 'cope{contrast_id}.nii.gz'), + 'varcope' : join(self.directories.output_dir, + 'l1_analysis', '_run_id_*_subject_id_{subject_id}', 'results', + 'varcope{contrast_id}.nii.gz'), + + ##### TODO not dataset here + 'mask': join(self.directories.dataset_dir, 'NARPS-T54A', 'hypo1_cope.nii.gz') + } + + # SelectFiles node - to select necessary files + selectfiles_sub_level = Node(SelectFiles( + templates, base_directory = self.directories.results_dir), + name = 'selectfiles_sub_level') + + # Datasink - save important files + datasink_sub_level = Node(DataSink( + base_directory = str(self.directories.output_dir) + ), + name = 'datasink_sub_level') + + # Generate design matrix + specify_model_sub_level = Node(L2Model( + num_copes = len(self.run_list)), + name = 'specify_model_sub_level') + + # Merge copes and varcopes files for each subject + merge_copes = Node(Merge(dimension = 't'), + name='merge_copes') + merge_varcopes = Node(Merge(dimension='t'), + name='merge_varcopes') + + flame = Node(FLAMEO(run_mode = 'flame1'), + name='flameo') + + # Second level (single-subject, mean of all four scans) analyses: Fixed effects analysis. + l2_analysis = Workflow( + base_dir = self.directories.working_dir, + name = 'l2_analysis') + l2_analysis.connect([ + (infosource_sub_level, selectfiles_sub_level, [ + ('subject_id', 'subject_id'), + ('contrast_id', 'contrast_id')]), + (selectfiles_sub_level, merge_copes, [('cope', 'in_files')]), + (selectfiles_sub_level, merge_varcopes, [('varcope', 'in_files')]), + (selectfiles_sub_level, flame, [('mask', 'mask_file')]), + (merge_copes, flame, [('merged_file', 'cope_file')]), + (merge_varcopes, flame, [('merged_file', 'var_cope_file')]), + (specify_model_sub_level, flame, [ + ('design_mat', 'design_file'), + ('design_con', 't_con_file'), + ('design_grp', 'cov_split_file')]), + (flame, datasink_sub_level, [ + ('zstats', 'l2_analysis.@stats'), + ('tstats', 'l2_analysis.@tstats'), + ('copes', 'l2_analysis.@copes'), + ('var_copes', 'l2_analysis.@varcopes')])]) + + return l2_analysis + + def get_subject_level_outputs(self): + """ Return the names of the files the subject level analysis is supposed to generate. """ + + parameters = { + 'contrast_id' : self.contrast_list, + 'subject_id' : self.subject_list, + 'file' : ['cope1.nii.gz', 'tstat1.nii.gz', 'varcope1.nii.gz', 'zstat1.nii.gz'] + } + parameter_sets = product(*parameters.values()) + template = join( + self.directories.output_dir, + 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}','{file}' + ) + + return [template.format(**dict(zip(parameters.keys(), parameter_values)))\ + for parameter_values in parameter_sets] + + def get_subgroups_contrasts(copes, varcopes, subject_ids, participants_file): + """ + Parameters : + - copes: original file list selected by selectfiles node + - varcopes: original file list selected by selectfiles node + - subject_ids: list of subject IDs that are analyzed + - participants_file: str, file containing participants characteristics + + Returns : the file list containing only the files belonging to subject in the wanted group. + """ + equal_range_id = [] + equal_indifference_id = [] + subject_list = [f'sub-{str(s).zfill(3)}' for s in subject_ids] + + with open(participants_file, 'rt') as file: + next(file) # skip the header + + for line in file: + info = line.strip().split() + if info[0] in subject_list and info[1] == 'equalIndifference': + equal_indifference_id.append(info[0][-3:]) + elif info[0] in subject_list and info[1] == 'equalRange': + equal_range_id.append(info[0][-3:]) + + copes_equal_indifference = [] + copes_equal_range = [] + copes_global = [] + varcopes_equal_indifference = [] + varcopes_equal_range = [] + varcopes_global = [] + + for file in copes: + sub_id = file.split('/') + if sub_id[-1][4:7] in equal_indifference_id: + copes_equal_indifference.append(file) + elif sub_id[-1][4:7] in equal_range_id: + copes_equal_range.append(file) + if sub_id[-1][4:7] in subject_ids: + copes_global.append(file) + + for file in varcopes: + sub_id = file.split('/') + if sub_id[-1][4:7] in equal_indifference_id: + varcopes_equal_indifference.append(file) + elif sub_id[-1][4:7] in equal_range_id: + varcopes_equal_range.append(file) + if sub_id[-1][4:7] in subject_ids: + varcopes_global.append(file) + + return copes_equal_indifference, copes_equal_range,\ + varcopes_equal_indifference, varcopes_equal_range,\ + equal_indifference_id, equal_range_id,\ + copes_global, varcopes_global + + def get_regressors(equal_range_ids, equal_indifference_ids, method, subject_list): + """ + Create dictionary of regressors for group analysis. + + Parameters: + - equal_range_ids: list of str, ids of subjects in equal range group + - equal_indifference_ids: list of str, ids of subjects in equal indifference group + - method: one of 'equalRange', 'equalIndifference' or 'groupComp' + - subject_list: list of str, ids of subject for which to do the analysis + + Returns: + - regressors: dict, dictionary of regressors used to + distinguish groups in FSL group analysis + """ + if method == 'equalRange': + regressors = dict(group_mean = [1 for i in range(len(equal_range_ids))]) + elif method == 'equalIndifference': + regressors = dict(group_mean = [1 for i in range(len(equal_indifference_ids))]) + elif method == 'groupComp': + equal_range_reg = [ + 1 for i in range(len(equal_range_ids) + len(equal_indifference_ids)) + ] + equal_indifference_reg = [ + 0 for i in range(len(equal_range_ids) + len(equal_indifference_ids)) + ] + + for index, sub_id in enumerate(subject_list): + if sub_id in equal_indifference_ids: + equal_indifference_reg[index] = 1 + equal_range_reg[index] = 0 + + regressors = dict( + equalRange = equal_range_reg, + equalIndifference = equal_indifference_reg + ) + + return regressors + + def get_group_level_analysis(self): + """ + Return all workflows for the group level analysis. + + Returns; + - a list of nipype.WorkFlow + """ + + methods = ['equalRange', 'equalIndifference', 'groupComp'] + return [self.get_group_level_analysis_sub_workflow(method) for method in methods] + + def get_group_level_analysis_sub_workflow(self, method): + """ + Return a workflow for the group level analysis. + + Parameters: + - method: one of 'equalRange', 'equalIndifference' or 'groupComp' + + Returns: + - l3_analysis: nipype.WorkFlow + """ + # Compute the number of participants used to do the analysis + nb_subjects = len(self.subject_list) + + # Infosource Node - To iterate on subject and runs + infosource_group_level = Node(IdentityInterface( + fields = ['contrast_ids', 'subject_list'], + subject_list = self.subject_list), + name = 'infosource_group_level') + infosource_group_level.iterables = [('contrast_id', self.contrast_list)] + + # Templates to select files node + templates = { + ##### TODO not dataset here + 'cope' : join(self.directories.output_dir, + 'l2_analysis', '_contrast-{contrast_id}_subject_id_cope.nii.gz'), + ##### TODO not dataset here + 'varcope' : join(data_dir, 'NARPS-T54A', 'sub-*_contrast-{contrast_id}_varcope.nii.gz'), + 'participants' : join(self.directories.dataset_dir, 'participants.tsv'), + ##### TODO not dataset here + 'mask' : join(data_dir, 'NARPS-T54A', 'hypo2_unthresh_Z.nii.gz') + } + + # SelectFiles node - to select necessary files + selectfiles_group_level = Node(SelectFiles( + templates, base_directory = self.directories.result_dir), + name = 'selectfiles_group_level') + + datasink_group_level = Node(DataSink( + base_directory = self.directories.result_dir + ), + name='datasink_group_level') + + merge_copes_group_level = Node(Merge(dimension = 't'), + name = 'merge_copes_group_level') + merge_varcopes_group_level = Node(Merge(dimension = 't'), + name = 'merge_varcopes_group_level') + + subgroups_contrasts = Node(Function( + function = self.get_subgroups_contrasts, + input_names = ['copes', 'varcopes', 'subject_list', 'participants_file'], + output_names = [ + 'copes_equalIndifference', 'copes_equalRange', + 'varcopes_equalIndifference', 'varcopes_equalRange', + 'equalIndifference_id', 'equalRange_id', + 'copes_global', 'varcopes_global'] + ), + name = 'subgroups_contrasts') + + specifymodel_group_level = Node(MultipleRegressDesign(), + name = 'specifymodel_group_level') + + flame_group_level = Node(FLAMEO(run_mode = 'flame1'), + name='flame_group_level') + + regressors = Node(Function( + function = self.get_regressors, + input_names = ['equalRange_id', 'equalIndifference_id', 'method', 'subject_list'], + output_names = ['regressors']), + name = 'regressors') + regressors.inputs.method = method + regressors.inputs.subject_list = self.subject_list + + randomise = Node(Randomise( + num_perm = 10000, tfce = True, vox_p_values = True, c_thresh = 0.05, tfce_E = 0.01), + name = "randomise") + + l3_analysis = Workflow( + base_dir = self.directories.working_dir, + name = f'l3_analysis_{method}_nsub_{nb_subjects}') + l3_analysis.connect([ + (infosource_group_level, selectfiles_group_level, [('contrast_id', 'contrast_id')]), + (infosource_group_level, subgroups_contrasts, [('subject_list', 'subject_ids')]), + (selectfiles_group_level, subgroups_contrasts, [ + ('cope', 'copes'), + ('varcope', 'varcopes'), + ('participants', 'participants_file')]), + (selectfiles_group_level, flame_group_level, [('mask', 'mask_file')]), + (selectfiles_group_level, randomise, [('mask', 'mask')]), + (subgroups_contrasts, regressors, [ + ('equalRange_id', 'equalRange_id'), + ('equalIndifference_id', 'equalIndifference_id')]), + (regressors, specifymodel_group_level, [('regressors', 'regressors')])]) + + if method in ('equalIndifference', 'equalRange'): + specifymodel_group_level.inputs.contrasts = [ + ['group_mean', 'T', ['group_mean'], [1]], + ['group_mean_neg', 'T', ['group_mean'], [-1]] + ] + + if method == 'equalIndifference': + l3_analysis.connect([ + (subgroups_contrasts, merge_copes_group_level, + [('copes_equalIndifference', 'in_files')] + ), + (subgroups_contrasts, merge_varcopes_group_level, + [('varcopes_equalIndifference', 'in_files')] + ) + ]) + elif method == 'equalRange': + l3_analysis.connect([ + (subgroups_contrasts, merge_copes_group_level, + [('copes_equalRange', 'in_files')] + ), + (subgroups_contrasts, merge_varcopes_group_level, + [('varcopes_equalRange', 'in_files')] + ) + ]) + + elif method == 'groupComp': + specifymodel_group_level.inputs.contrasts = [ + ['equalRange_sup', 'T', ['equalRange', 'equalIndifference'], [1, -1]] + ] + l3_analysis.connect([ + (subgroups_contrasts, merge_copes_group_level, + [('copes_global', 'in_files')] + ), + (subgroups_contrasts, merge_varcopes_group_level, + [('varcopes_global', 'in_files')] + ) + ]) + + l3_analysis.connect([ + (merge_copes_group_level, flame_group_level, [('merged_file', 'cope_file')]), + (merge_varcopes_group_level, flame_group_level, [('merged_file', 'var_cope_file')]), + (specifymodel_group_level, flame_group_level, [ + ('design_mat', 'design_file'), + ('design_con', 't_con_file'), + ('design_grp', 'cov_split_file')]), + (merge_copes_group_level, randomise, [('merged_file', 'in_file')]), + (specifymodel_group_level, randomise, [ + ('design_mat', 'design_mat'), + ('design_con', 'tcon')]), + (randomise, datasink_group_level, [ + ('t_corrected_p_files', f'l3_analysis_{method}_nsub_{nb_subjects}.@tcorpfile'), + ('tstat_files', f'l3_analysis_{method}_nsub_{nb_subjects}.@tstat')]), + (flame_group_level, datasink_group_level, [ + ('zstats', f'l3_analysis_{method}_nsub_{nb_subjects}.@zstats'), + ('tstats', f'l3_analysis_{method}_nsub_{nb_subjects}.@tstats')]), + ]) + + return l3_analysis + + def get_group_level_outputs(self): + """ Return all names for the files the group level analysis is supposed to generate. """ + + # Handle equalRange and equalIndifference + parameters = { + 'contrast_id': ['ploss', 'pgain'], + 'method': ['equalRange', 'equalIndifference'], + 'file': [ + 'randomise_tfce_corrp_tstat1.nii.gz', + 'randomise_tfce_corrp_tstat2.nii.gz', + 'randomise_tstat1.nii.gz', + 'randomise_tstat2.nii.gz', + 'tstat1.nii.gz', + 'tstat2.nii.gz', + 'zstat1.nii.gz', + 'zstat2.nii.gz' + ], + 'nb_subjects' : [str(len(self.subject_list))] + } + parameter_sets = product(*parameters.values()) + template = join( + self.directories.output_dir, + 'l3_analysis_{method}_nsub_{nb_subjects}', + '_contrast_id_{contrast_id}', + '{file}' + ) + + return_list = [template.format(**dict(zip(parameters.keys(), parameter_values)))\ + for parameter_values in parameter_sets] + + # Handle groupComp + files : [ + 'randomise_tfce_corrp_tstat1.nii.gz', + 'randomise_tstat1.nii.gz', + 'zstat1.nii.gz', + 'tstat1.nii.gz' + ] + + return_list += [join( + self.directories.output_dir, + f'l3_analysis_groupComp_nsub_{len(self.subject_list)}', + '_contrast_id_ploss', f'{file}') for file in files] + + return return_list + + def get_hypotheses_outputs(self): + """ Return all hypotheses output file names. """ + + nb_sub = len(self.subject_list) + files = [ + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat2.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat2.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat2.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat2.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat1.nii.gz'), + join(f'l3_analysis_groupComp_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_groupComp_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat1.nii.gz') + ] + return [join(self.directories.output_dir, f) for f in files] + +##### TODO : what is this ? +system('export PATH=$PATH:/local/egermani/ICA-AROMA') diff --git a/tests/pipelines/test_team_T54A.py b/tests/pipelines/test_team_T54A.py new file mode 100644 index 00000000..73ee1688 --- /dev/null +++ b/tests/pipelines/test_team_T54A.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# coding: utf-8 + +""" Tests of the 'narps_open.pipelines.team_T54A' module. + +Launch this test with PyTest + +Usage: +====== + pytest -q test_team_T54A.py + pytest -q test_team_T54A.py -k +""" + +from pytest import helpers, mark +from nipype import Workflow + +from narps_open.pipelines.team_T54A import PipelineTeamT54A + +class TestPipelinesTeamT54A: + """ A class that contains all the unit tests for the PipelineTeamT54A class.""" + + @staticmethod + @mark.unit_test + def test_create(): + """ Test the creation of a PipelineTeamT54A object """ + + pipeline = PipelineTeamT54A() + + # 1 - check the parameters + assert pipeline.fwhm == 8.0 + assert pipeline.team_id == 'T54A' + + # 2 - check workflows + assert pipeline.get_preprocessing() is None + assert pipeline.get_run_level_analysis() is None + assert isinstance(pipeline.get_subject_level_analysis(), Workflow) + group_level = pipeline.get_group_level_analysis() + + assert len(group_level) == 3 + for sub_workflow in group_level: + assert isinstance(sub_workflow, Workflow) + + @staticmethod + @mark.unit_test + def test_outputs(): + """ Test the expected outputs of a PipelineTeamT54A object """ + pipeline = PipelineTeamT54A() + # 1 - 1 subject outputs + pipeline.subject_list = ['001'] + assert len(pipeline.get_preprocessing_outputs()) == 0 + assert len(pipeline.get_run_level_outputs()) == 0 + assert len(pipeline.get_subject_level_outputs()) == 7 + assert len(pipeline.get_group_level_outputs()) == 63 + assert len(pipeline.get_hypotheses_outputs()) == 18 + + # 2 - 4 subjects outputs + pipeline.subject_list = ['001', '002', '003', '004'] + assert len(pipeline.get_preprocessing_outputs()) == 0 + assert len(pipeline.get_run_level_outputs()) == 0 + assert len(pipeline.get_subject_level_outputs()) == 28 + assert len(pipeline.get_group_level_outputs()) == 63 + assert len(pipeline.get_hypotheses_outputs()) == 18 + + @staticmethod + @mark.pipeline_test + def test_execution(): + """ Test the execution of a PipelineTeamT54A and compare results """ + helpers.test_pipeline_evaluation('T54A') From e8ec3aec9ab0fbfc90412f68d762453d335b0ed9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 29 Aug 2023 14:05:55 +0200 Subject: [PATCH 03/33] [TEST] test updates for T54A pipeline --- narps_open/pipelines/team_T54A.py | 54 ++++++++++++++++++++----------- tests/pipelines/test_team_T54A.py | 16 ++++----- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index b54a0ee7..eb96ea88 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -772,24 +772,42 @@ def get_hypotheses_outputs(self): nb_sub = len(self.subject_list) files = [ - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat2.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat2.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat2.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat2.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat1.nii.gz'), - join(f'l3_analysis_groupComp_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_groupComp_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat1.nii.gz') + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_pgain', 'zstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_pgain', 'zstat1.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_pgain', 'zstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_pgain', 'zstat1.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_ploss', 'randomise_tfce_corrp_tstat2.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_ploss', 'zstat2.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_ploss', 'randomise_tfce_corrp_tstat2.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_ploss', 'zstat2.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_ploss', 'zstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_ploss', 'zstat1.nii.gz'), + join(f'l3_analysis_groupComp_nsub_{nb_sub}', + '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), + join(f'l3_analysis_groupComp_nsub_{nb_sub}', + '_contrast_id_ploss', 'zstat1.nii.gz') ] return [join(self.directories.output_dir, f) for f in files] diff --git a/tests/pipelines/test_team_T54A.py b/tests/pipelines/test_team_T54A.py index 73ee1688..0c3c2640 100644 --- a/tests/pipelines/test_team_T54A.py +++ b/tests/pipelines/test_team_T54A.py @@ -27,12 +27,12 @@ def test_create(): pipeline = PipelineTeamT54A() # 1 - check the parameters - assert pipeline.fwhm == 8.0 + assert pipeline.fwhm == 4.0 assert pipeline.team_id == 'T54A' # 2 - check workflows assert pipeline.get_preprocessing() is None - assert pipeline.get_run_level_analysis() is None + assert isinstance(pipeline.get_run_level_analysis(), Workflow) assert isinstance(pipeline.get_subject_level_analysis(), Workflow) group_level = pipeline.get_group_level_analysis() @@ -48,17 +48,17 @@ def test_outputs(): # 1 - 1 subject outputs pipeline.subject_list = ['001'] assert len(pipeline.get_preprocessing_outputs()) == 0 - assert len(pipeline.get_run_level_outputs()) == 0 - assert len(pipeline.get_subject_level_outputs()) == 7 - assert len(pipeline.get_group_level_outputs()) == 63 + assert len(pipeline.get_run_level_outputs()) == 33*4*1 + assert len(pipeline.get_subject_level_outputs()) == 4*2*1 + assert len(pipeline.get_group_level_outputs()) == 8*2*2 + 4 assert len(pipeline.get_hypotheses_outputs()) == 18 # 2 - 4 subjects outputs pipeline.subject_list = ['001', '002', '003', '004'] assert len(pipeline.get_preprocessing_outputs()) == 0 - assert len(pipeline.get_run_level_outputs()) == 0 - assert len(pipeline.get_subject_level_outputs()) == 28 - assert len(pipeline.get_group_level_outputs()) == 63 + assert len(pipeline.get_run_level_outputs()) == 33*4*4 + assert len(pipeline.get_subject_level_outputs()) == 4*2*4 + assert len(pipeline.get_group_level_outputs()) == 8*2*2 + 4 assert len(pipeline.get_hypotheses_outputs()) == 18 @staticmethod From 0a4eadd71c685e7d6286b7f70898e157535934cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 29 Aug 2023 14:29:34 +0200 Subject: [PATCH 04/33] Remove changes on 2T6S --- narps_open/pipelines/team_2T6S.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/narps_open/pipelines/team_2T6S.py b/narps_open/pipelines/team_2T6S.py index 0ec99d75..42aa344f 100755 --- a/narps_open/pipelines/team_2T6S.py +++ b/narps_open/pipelines/team_2T6S.py @@ -1,7 +1,7 @@ #!/usr/bin/python # coding: utf-8 -""" Write the work of NARPS team 2T6S using Nipype """ +""" Write the work of NARPS' team 2T6S using Nipype """ from os.path import join from itertools import product From 69fd60b0b5fbe611e48643d8acfa9f9aaf064b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 29 Aug 2023 14:42:21 +0200 Subject: [PATCH 05/33] [TEST] change pipeline test_utils to avoid having to rewrite it at every repro --- tests/pipelines/test_pipelines.py | 8 +- tests/test_conftest.py | 139 ++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 tests/test_conftest.py diff --git a/tests/pipelines/test_pipelines.py b/tests/pipelines/test_pipelines.py index 9016aeb7..c38cf36a 100644 --- a/tests/pipelines/test_pipelines.py +++ b/tests/pipelines/test_pipelines.py @@ -135,8 +135,8 @@ class TestUtils: @mark.unit_test def test_utils(): """ Test the utils methods of PipelineRunner """ - # 1 - Get number of not implemented pipelines - assert len(get_not_implemented_pipelines()) == 69 + # 1 - Get not implemented pipelines + assert '1K0E' in get_not_implemented_pipelines() - # 2 - Get number of implemented pipelines - assert len(get_implemented_pipelines()) == 1 + # 2 - Get implemented pipelines + assert '2T6S' in get_implemented_pipelines() diff --git a/tests/test_conftest.py b/tests/test_conftest.py new file mode 100644 index 00000000..5b1a1f4f --- /dev/null +++ b/tests/test_conftest.py @@ -0,0 +1,139 @@ +#!/usr/bin/python +# coding: utf-8 + +""" Tests of the 'conftest.py' module. + +Launch this test with PyTest + +Usage: +====== + pytest -q test_conftest.py + pytest -q test_conftest.py -k +""" + +from os import remove +from os.path import join, isfile, abspath +from pathlib import Path + +from datetime import datetime + +from pytest import raises, mark + +from nipype import Node, Workflow +from nipype.interfaces.utility import Function + +from narps_open.utils.configuration import Configuration +from narps_open.runner import PipelineRunner +from narps_open.pipelines import Pipeline +from narps_open.pipelines.team_2T6S import PipelineTeam2T6S + +class MockupPipeline(Pipeline): + """ A simple Pipeline class for test purposes """ + + def __init__(self): + super().__init__() + self.test_file = abspath( + join(Configuration()['directories']['test_runs'], 'test_conftest.txt')) + if isfile(self.test_file): + remove(self.test_file) + + def __del__(self): + if isfile(self.test_file): + remove(self.test_file) + + # @staticmethod + def write_to_file(_, text_to_write: str, file_path: str): + """ Method used inside a nipype Node, to write a line in a test file """ + with open(file_path, 'a', encoding = 'utf-8') as file: + file.write(text_to_write) + + def create_workflow(self, workflow_name: str): + """ Return a nipype workflow with two nodes writing in a file """ + node_1 = Node(Function( + input_names = ['_', 'text_to_write', 'file_path'], + output_names = ['_'], + function = self.write_to_file), + name = 'node_1' + ) + # this input is set to now(), so that it changes at every run, thus preventing + # nipype's cache to work + node_1.inputs._ = datetime.now() + node_1.inputs.text_to_write = 'MockupPipeline : '+workflow_name+' node_1\n' + node_1.inputs.file_path = self.test_file + + node_2 = Node(Function( + input_names = ['_', 'text_to_write', 'file_path'], + output_names = [], + function = self.write_to_file), + name = 'node_2' + ) + node_2.inputs.text_to_write = 'MockupPipeline : '+workflow_name+' node_2\n' + node_2.inputs.file_path = self.test_file + + workflow = Workflow( + base_dir = Configuration()['directories']['test_runs'], + name = workflow_name + ) + workflow.add_nodes([node_1, node_2]) + workflow.connect(node_1, '_', node_2, '_') + + return workflow + + def get_preprocessing(self): + """ Return a fake preprocessing workflow """ + return self.create_workflow('TestPipelineRunner_preprocessing_workflow') + + def get_run_level_analysis(self): + """ Return a fake run level workflow """ + return self.create_workflow('TestPipelineRunner_run_level_workflow') + + def get_subject_level_analysis(self): + """ Return a fake subject level workflow """ + return self.create_workflow('TestPipelineRunner_subject_level_workflow') + + def get_group_level_analysis(self): + """ Return a fake subject level workflow """ + return self.create_workflow('TestPipelineRunner_group_level_workflow') + + def get_preprocessing_outputs(self): + """ Return a list of templates of the output files generated by the preprocessing """ + return [join(Configuration()['directories']['test_runs'], 'preprocessing_output.md')] + + def get_run_level_outputs(self): + """ Return a list of templates of the output files generated by the run level analysis. + Templates are expressed relatively to the self.directories.output_dir. + """ + return [join(Configuration()['directories']['test_runs'], 'run_output.md')] + + def get_subject_level_outputs(self): + """ Return a list of templates of the output files generated by the subject level analysis. + Templates are expressed relatively to the self.directories.output_dir. + """ + templates = [ + join(Configuration()['directories']['test_runs'], 'subject_{subject_id}_output_1.md'), + join(Configuration()['directories']['test_runs'], 'subject_{subject_id}_output_2.md') + ] + return_list = [] + for subject_id in self.subject_list: + return_list += [t.format(subject_id = subject_id) for t in templates] + + return return_list + + def get_group_level_outputs(self): + """ Return a list of templates of the output files generated by the group level analysis. + Templates are expressed relatively to the self.directories.output_dir. + """ + templates = [ + join(Configuration()['directories']['test_runs'], 'group_{nb_subjects}_output_a.md'), + join(Configuration()['directories']['test_runs'], 'group_{nb_subjects}_output_b.md') + ] + return [t.format(nb_subjects = len(self.subject_list)) for t in templates] + + def get_hypotheses_outputs(self): + """ Return the names of the files used by the team to answer the hypotheses of NARPS. + """ + template = join(Configuration()['directories']['test_runs'], 'hypothesis_{id}.md') + return [template.format(id = i) for i in range(1,18)] + +class TestPipelineRunner: + """ A class that contains all the unit tests for the PipelineRunner class.""" From 6640ca367c85fe4951f8e4b90643370832de1dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 29 Aug 2023 16:41:02 +0200 Subject: [PATCH 06/33] [CI] from now on, unit_test workflow must run on self hosted runner --- .github/workflows/unit_tests.yml | 15 +++------------ narps_open/pipelines/team_T54A.py | 8 ++++---- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 4859e55d..e0c237b4 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -27,24 +27,15 @@ jobs: pytest: # Define the runner for this job - runs-on: ubuntu-latest + runs-on: self-hosted # Steps that define the job steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up Python 3.9 - uses: actions/setup-python@v3 - with: - python-version: 3.9 - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + - name: Load configuration for self-hosted runner + run: cp /home/neuro/local_testing_config.toml narps_open/utils/configuration/testing_config.toml - name: Install dependencies run: | diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index eb96ea88..426b1b45 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -228,7 +228,7 @@ def get_run_level_analysis(self): name = 'skullstrip') # Smoothing - smooth = Node(IsotropicSmooth(fwhm = self.fwhm), + smooth = Node(IsotropicSmooth(fwhm = self.fwhm), # TODO : Previously set to 6 mm ? name = 'smooth') # Function node get_subject_infos - get subject specific condition information @@ -254,8 +254,8 @@ def get_run_level_analysis(self): parameters = Node(Function( function = self.get_parameters_file, input_names = ['filepath', 'subject_id', 'run_id', 'working_dir'], - output_names=['parameters_file']), - name='parameters') + output_names = ['parameters_file']), + name = 'parameters') parameters.inputs.working_dir = self.directories.working_dir # First temporal derivatives of the two regressors were also used, @@ -753,7 +753,7 @@ def get_group_level_outputs(self): for parameter_values in parameter_sets] # Handle groupComp - files : [ + files = [ 'randomise_tfce_corrp_tstat1.nii.gz', 'randomise_tstat1.nii.gz', 'zstat1.nii.gz', From 89a2d68f5a40d3bfdfe8df3077d9196a68a70ba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 29 Aug 2023 17:42:08 +0200 Subject: [PATCH 07/33] [BUG] Cleaning connections --- narps_open/pipelines/team_T54A.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 426b1b45..91547936 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -246,7 +246,7 @@ def get_run_level_analysis(self): # Node contrasts to get contrasts contrasts = Node(Function( function = self.get_contrasts, - input_names = ['subject_id'], + input_names = [], output_names = ['contrasts']), name = 'contrasts') @@ -289,7 +289,6 @@ def get_run_level_analysis(self): ('run_id', 'run_id')]), (selectfiles, subject_infos, [('event', 'event_file')]), (selectfiles, parameters, [('param', 'filepath')]), - (infosource, contrasts, [('subject_id', 'subject_id')]), (infosource, parameters, [ ('subject_id', 'subject_id'), ('run_id', 'run_id')]), @@ -592,11 +591,10 @@ def get_group_level_analysis_sub_workflow(self, method): # Templates to select files node templates = { - ##### TODO not dataset here 'cope' : join(self.directories.output_dir, - 'l2_analysis', '_contrast-{contrast_id}_subject_id_cope.nii.gz'), - ##### TODO not dataset here - 'varcope' : join(data_dir, 'NARPS-T54A', 'sub-*_contrast-{contrast_id}_varcope.nii.gz'), + 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', 'cope1.nii.gz'), + 'varcope' : join(self.directories.output_dir, + 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', 'varcope1.nii.gz'), 'participants' : join(self.directories.dataset_dir, 'participants.tsv'), ##### TODO not dataset here 'mask' : join(data_dir, 'NARPS-T54A', 'hypo2_unthresh_Z.nii.gz') From f9c7ab0f5732d7853d6b8f36aa27dc0bc4df70e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Wed, 30 Aug 2023 10:30:12 +0200 Subject: [PATCH 08/33] [TEST] test_conftest was not supposed to belong to this branch --- tests/test_conftest.py | 139 ----------------------------------------- 1 file changed, 139 deletions(-) delete mode 100644 tests/test_conftest.py diff --git a/tests/test_conftest.py b/tests/test_conftest.py deleted file mode 100644 index 5b1a1f4f..00000000 --- a/tests/test_conftest.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/python -# coding: utf-8 - -""" Tests of the 'conftest.py' module. - -Launch this test with PyTest - -Usage: -====== - pytest -q test_conftest.py - pytest -q test_conftest.py -k -""" - -from os import remove -from os.path import join, isfile, abspath -from pathlib import Path - -from datetime import datetime - -from pytest import raises, mark - -from nipype import Node, Workflow -from nipype.interfaces.utility import Function - -from narps_open.utils.configuration import Configuration -from narps_open.runner import PipelineRunner -from narps_open.pipelines import Pipeline -from narps_open.pipelines.team_2T6S import PipelineTeam2T6S - -class MockupPipeline(Pipeline): - """ A simple Pipeline class for test purposes """ - - def __init__(self): - super().__init__() - self.test_file = abspath( - join(Configuration()['directories']['test_runs'], 'test_conftest.txt')) - if isfile(self.test_file): - remove(self.test_file) - - def __del__(self): - if isfile(self.test_file): - remove(self.test_file) - - # @staticmethod - def write_to_file(_, text_to_write: str, file_path: str): - """ Method used inside a nipype Node, to write a line in a test file """ - with open(file_path, 'a', encoding = 'utf-8') as file: - file.write(text_to_write) - - def create_workflow(self, workflow_name: str): - """ Return a nipype workflow with two nodes writing in a file """ - node_1 = Node(Function( - input_names = ['_', 'text_to_write', 'file_path'], - output_names = ['_'], - function = self.write_to_file), - name = 'node_1' - ) - # this input is set to now(), so that it changes at every run, thus preventing - # nipype's cache to work - node_1.inputs._ = datetime.now() - node_1.inputs.text_to_write = 'MockupPipeline : '+workflow_name+' node_1\n' - node_1.inputs.file_path = self.test_file - - node_2 = Node(Function( - input_names = ['_', 'text_to_write', 'file_path'], - output_names = [], - function = self.write_to_file), - name = 'node_2' - ) - node_2.inputs.text_to_write = 'MockupPipeline : '+workflow_name+' node_2\n' - node_2.inputs.file_path = self.test_file - - workflow = Workflow( - base_dir = Configuration()['directories']['test_runs'], - name = workflow_name - ) - workflow.add_nodes([node_1, node_2]) - workflow.connect(node_1, '_', node_2, '_') - - return workflow - - def get_preprocessing(self): - """ Return a fake preprocessing workflow """ - return self.create_workflow('TestPipelineRunner_preprocessing_workflow') - - def get_run_level_analysis(self): - """ Return a fake run level workflow """ - return self.create_workflow('TestPipelineRunner_run_level_workflow') - - def get_subject_level_analysis(self): - """ Return a fake subject level workflow """ - return self.create_workflow('TestPipelineRunner_subject_level_workflow') - - def get_group_level_analysis(self): - """ Return a fake subject level workflow """ - return self.create_workflow('TestPipelineRunner_group_level_workflow') - - def get_preprocessing_outputs(self): - """ Return a list of templates of the output files generated by the preprocessing """ - return [join(Configuration()['directories']['test_runs'], 'preprocessing_output.md')] - - def get_run_level_outputs(self): - """ Return a list of templates of the output files generated by the run level analysis. - Templates are expressed relatively to the self.directories.output_dir. - """ - return [join(Configuration()['directories']['test_runs'], 'run_output.md')] - - def get_subject_level_outputs(self): - """ Return a list of templates of the output files generated by the subject level analysis. - Templates are expressed relatively to the self.directories.output_dir. - """ - templates = [ - join(Configuration()['directories']['test_runs'], 'subject_{subject_id}_output_1.md'), - join(Configuration()['directories']['test_runs'], 'subject_{subject_id}_output_2.md') - ] - return_list = [] - for subject_id in self.subject_list: - return_list += [t.format(subject_id = subject_id) for t in templates] - - return return_list - - def get_group_level_outputs(self): - """ Return a list of templates of the output files generated by the group level analysis. - Templates are expressed relatively to the self.directories.output_dir. - """ - templates = [ - join(Configuration()['directories']['test_runs'], 'group_{nb_subjects}_output_a.md'), - join(Configuration()['directories']['test_runs'], 'group_{nb_subjects}_output_b.md') - ] - return [t.format(nb_subjects = len(self.subject_list)) for t in templates] - - def get_hypotheses_outputs(self): - """ Return the names of the files used by the team to answer the hypotheses of NARPS. - """ - template = join(Configuration()['directories']['test_runs'], 'hypothesis_{id}.md') - return [template.format(id = i) for i in range(1,18)] - -class TestPipelineRunner: - """ A class that contains all the unit tests for the PipelineRunner class.""" From 582a024193c2a2a1e79fd7b0e44b27662ca17374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Wed, 30 Aug 2023 15:20:14 +0200 Subject: [PATCH 09/33] Wildcard to create parameters dir --- narps_open/pipelines/team_T54A.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 91547936..efad45e7 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -113,7 +113,7 @@ def get_parameters_file(filepath, subject_id, run_id, working_dir): Return : - parameters_file : paths to new files containing only desired parameters. """ - from os import mkdir + from os import makedirs from os.path import join, isdir from pandas import read_csv, DataFrame @@ -134,8 +134,7 @@ def get_parameters_file(filepath, subject_id, run_id, working_dir): parameters_file = join(working_dir, 'parameters_file', f'parameters_file_sub-{subject_id}_run{run_id}.tsv') - if not isdir(join(working_dir, 'parameters_file')): - mkdir(join(working_dir, 'parameters_file')) + makedirs(join(working_dir, 'parameters_file'), exist_ok = True) with open(parameters_file, 'w') as writer: writer.write(retained_parameters.to_csv( From 5eaac52130b655d0e7cb93e877d88f2517269293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 31 Aug 2023 09:10:51 +0200 Subject: [PATCH 10/33] Set default file type for FSL --- narps_open/pipelines/team_T54A.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index efad45e7..180267aa 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -12,12 +12,15 @@ from nipype.interfaces.io import SelectFiles, DataSink from nipype.interfaces.fsl import ( BET, IsotropicSmooth, Level1Design, FEATModel, L2Model, Merge, FLAMEO, - FILMGLS, Randomise, MultipleRegressDesign + FILMGLS, Randomise, MultipleRegressDesign, FSLCommand ) from nipype.algorithms.modelgen import SpecifyModel from narps_open.pipelines import Pipeline +# Setup FSL +FSLCommand.set_default_output_type('NIFTI_GZ') + class PipelineTeamT54A(Pipeline): """ A class that defines the pipeline of team T54A. """ From 6986e90351bca4071ea61387bee1f52f1125aceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 31 Aug 2023 14:35:10 +0200 Subject: [PATCH 11/33] [BUG] inside unit_tests workflow --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 20f20ea3..d0097882 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -34,7 +34,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - name: Load configuration for self-hosted runner + - name: Load configuration for self-hosted runner run: cp /home/neuro/local_testing_config.toml narps_open/utils/configuration/testing_config.toml - name: Install dependencies From 7f174467d4951b929146145160cb58ee46261302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Fri, 29 Sep 2023 16:48:33 +0200 Subject: [PATCH 12/33] Use computed masks --- narps_open/pipelines/team_T54A.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 180267aa..96f39bb0 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -78,7 +78,7 @@ def get_session_infos(event_file): onset[condition].append(float(info[0])) duration[condition].append(float(info[4])) amplitude[condition].append(float(1)) - elif condition == 'difficonditionulty': + elif condition == 'difficulty': onset[condition].append(float(info[0])) duration[condition].append(float(info[4])) amplitude[condition].append( @@ -396,9 +396,9 @@ def get_subject_level_analysis(self): 'varcope' : join(self.directories.output_dir, 'l1_analysis', '_run_id_*_subject_id_{subject_id}', 'results', 'varcope{contrast_id}.nii.gz'), - - ##### TODO not dataset here - 'mask': join(self.directories.dataset_dir, 'NARPS-T54A', 'hypo1_cope.nii.gz') + 'mask': join(self.directories.output_dir, + 'l1_analysis', '_run_id_*_subject_id_{subject_id}', + 'sub-{subject_id}_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') } # SelectFiles node - to select necessary files @@ -598,8 +598,9 @@ def get_group_level_analysis_sub_workflow(self, method): 'varcope' : join(self.directories.output_dir, 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', 'varcope1.nii.gz'), 'participants' : join(self.directories.dataset_dir, 'participants.tsv'), - ##### TODO not dataset here - 'mask' : join(data_dir, 'NARPS-T54A', 'hypo2_unthresh_Z.nii.gz') + 'mask': join(self.directories.output_dir, + 'l1_analysis', '_run_id_*_subject_id_{subject_id}', + 'sub-{subject_id}_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') } # SelectFiles node - to select necessary files @@ -812,4 +813,4 @@ def get_hypotheses_outputs(self): return [join(self.directories.output_dir, f) for f in files] ##### TODO : what is this ? -system('export PATH=$PATH:/local/egermani/ICA-AROMA') +#system('export PATH=$PATH:/local/egermani/ICA-AROMA') From 442295b735ea255cd417ca847e5b1823db7f6ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 3 Oct 2023 11:23:20 +0200 Subject: [PATCH 13/33] Bug with output folder names --- narps_open/pipelines/team_T54A.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 96f39bb0..70a711c9 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -605,11 +605,11 @@ def get_group_level_analysis_sub_workflow(self, method): # SelectFiles node - to select necessary files selectfiles_group_level = Node(SelectFiles( - templates, base_directory = self.directories.result_dir), + templates, base_directory = self.directories.results_dir), name = 'selectfiles_group_level') datasink_group_level = Node(DataSink( - base_directory = self.directories.result_dir + base_directory = self.directories.output_dir ), name='datasink_group_level') From 3353a6d3d99f61238525264e5b765544a67ba2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 5 Oct 2023 16:57:18 +0200 Subject: [PATCH 14/33] PEP8 related refac --- narps_open/pipelines/team_T54A.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 70a711c9..e3efd6ac 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -3,7 +3,6 @@ """ Write the work of NARPS team T54A using Nipype """ -from os import system from os.path import join from itertools import product @@ -594,9 +593,11 @@ def get_group_level_analysis_sub_workflow(self, method): # Templates to select files node templates = { 'cope' : join(self.directories.output_dir, - 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', 'cope1.nii.gz'), + 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', + 'cope1.nii.gz'), 'varcope' : join(self.directories.output_dir, - 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', 'varcope1.nii.gz'), + 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', + 'varcope1.nii.gz'), 'participants' : join(self.directories.dataset_dir, 'participants.tsv'), 'mask': join(self.directories.output_dir, 'l1_analysis', '_run_id_*_subject_id_{subject_id}', @@ -645,7 +646,7 @@ def get_group_level_analysis_sub_workflow(self, method): randomise = Node(Randomise( num_perm = 10000, tfce = True, vox_p_values = True, c_thresh = 0.05, tfce_E = 0.01), - name = "randomise") + name = 'randomise') l3_analysis = Workflow( base_dir = self.directories.working_dir, From 6eed291f3029066a9eb9d76ad25e2dccc7a06bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Fri, 6 Oct 2023 12:02:31 +0200 Subject: [PATCH 15/33] Issue with contrasts ids --- narps_open/pipelines/team_T54A.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index e3efd6ac..3dbf01a6 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -27,7 +27,7 @@ def __init__(self): super().__init__() self.fwhm = 4.0 self.team_id = 'T54A' - self.contrast_list = ['01', '02'] + self.contrast_list = ['1', '2'] def get_preprocessing(self): """ No preprocessing has been done by team T54A """ @@ -326,9 +326,9 @@ def get_run_level_outputs(self): parameters = { 'run_id' : self.run_list, 'subject_id' : self.subject_list, + 'contrast_id' : self.contrast_list, 'file' : [ - join('results', 'cope1.nii.gz'), - join('results', 'cope2.nii.gz'), + join('results', 'cope{contrast_id}.nii.gz'), join('results', 'dof'), join('results', 'logfile'), join('results', 'pe10.nii.gz'), @@ -351,12 +351,9 @@ def get_run_level_outputs(self): join('results', 'res4d.nii.gz'), join('results', 'sigmasquareds.nii.gz'), join('results', 'threshac1.nii.gz'), - join('results', 'tstat1.nii.gz'), - join('results', 'tstat2.nii.gz'), - join('results', 'varcope1.nii.gz'), - join('results', 'varcope2.nii.gz'), - join('results', 'zstat1.nii.gz'), - join('results', 'zstat2.nii.gz'), + join('results', 'tstat{contrast_id}.nii.gz'), + join('results', 'varcope{contrast_id}.nii.gz'), + join('results', 'zstat{contrast_id}.nii.gz'), 'run0.mat', 'run0.png', 'sub-{subject_id}_task-MGT_run-{run_id}_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz' From f4713fd0112e34a47b239346aac6cb784fa01ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Mon, 9 Oct 2023 11:59:54 +0200 Subject: [PATCH 16/33] Run level outputs --- narps_open/pipelines/team_T54A.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 3dbf01a6..6035caf7 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -326,9 +326,7 @@ def get_run_level_outputs(self): parameters = { 'run_id' : self.run_list, 'subject_id' : self.subject_list, - 'contrast_id' : self.contrast_list, 'file' : [ - join('results', 'cope{contrast_id}.nii.gz'), join('results', 'dof'), join('results', 'logfile'), join('results', 'pe10.nii.gz'), @@ -351,9 +349,6 @@ def get_run_level_outputs(self): join('results', 'res4d.nii.gz'), join('results', 'sigmasquareds.nii.gz'), join('results', 'threshac1.nii.gz'), - join('results', 'tstat{contrast_id}.nii.gz'), - join('results', 'varcope{contrast_id}.nii.gz'), - join('results', 'zstat{contrast_id}.nii.gz'), 'run0.mat', 'run0.png', 'sub-{subject_id}_task-MGT_run-{run_id}_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz' @@ -364,10 +359,31 @@ def get_run_level_outputs(self): self.directories.output_dir, 'l1_analysis', '_run_id_{run_id}_subject_id_{subject_id}','{file}' ) + return_list = [template.format(**dict(zip(parameters.keys(), parameter_values)))\ + for parameter_values in parameter_sets] - return [template.format(**dict(zip(parameters.keys(), parameter_values)))\ + parameters = { + 'run_id' : self.run_list, + 'subject_id' : self.subject_list, + 'contrast_id' : self.contrast_list, + 'file' : [ + join('results', 'cope{contrast_id}.nii.gz'), + join('results', 'tstat{contrast_id}.nii.gz'), + join('results', 'varcope{contrast_id}.nii.gz'), + join('results', 'zstat{contrast_id}.nii.gz'), + ] + } + parameter_sets = product(*parameters.values()) + template = join( + self.directories.output_dir, + 'l1_analysis', '_run_id_{run_id}_subject_id_{subject_id}','{file}' + ) + + return_list += [template.format(**dict(zip(parameters.keys(), parameter_values)))\ for parameter_values in parameter_sets] + return return_list + def get_subject_level_analysis(self): """ Create the subject level analysis workflow. From c2950466fb47d4978b650d870db76521fcc526be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 16 Nov 2023 12:12:19 +0100 Subject: [PATCH 17/33] [REFAC] refactoring T54A + adding groups to MultipleRegressDesign [skip ci] --- narps_open/pipelines/team_T54A.py | 598 ++++++++++++++++-------------- 1 file changed, 323 insertions(+), 275 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 6035caf7..44daea20 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -16,6 +16,7 @@ from nipype.algorithms.modelgen import SpecifyModel from narps_open.pipelines import Pipeline +from narps_open.data.task import TaskInformation # Setup FSL FSLCommand.set_default_output_type('NIFTI_GZ') @@ -192,17 +193,18 @@ def get_run_level_analysis(self): Create the run level analysis workflow. Returns: - - l1_analysis : nipype.WorkFlow + - run_level_analysis : nipype.WorkFlow """ - # Infosource Node - To iterate on subject and runs - infosource = Node(IdentityInterface( + # IdentityInterface Node - To iterate on subject and runs + information_source = Node(IdentityInterface( fields = ['subject_id', 'run_id']), - name = 'infosource') - infosource.iterables = [ + name = 'information_source') + information_source.iterables = [ ('subject_id', self.subject_list), - ('run_id', self.run_list)] + ('run_id', self.run_list) + ] - # Templates to select files node + # SelectFiles - to select necessary files template = { # Parameter file 'param' : join('derivatives', 'fmriprep', 'sub-{subject_id}', 'func', @@ -215,43 +217,44 @@ def get_run_level_analysis(self): 'event' : join('sub-{subject_id}', 'func', 'sub-{subject_id}_task-MGT_run-{run_id}_events.tsv') } + select_files = Node(SelectFiles(template), name = 'select_files') + select_files.inputs.base_directory = self.directories.dataset_dir - # SelectFiles - to select necessary files - selectfiles = Node(SelectFiles(template, base_directory = self.directories.dataset_dir), - name = 'selectfiles') - - # DataSink - store the wanted results in the wanted repository - datasink = Node(DataSink(base_directory = self.directories.output_dir), - name='datasink') + # DataSink Node - store the wanted results in the wanted directory + data_sink = Node(DataSink(), name = 'data_sink') + data_sink.inputs.base_directory = self.directories.output_dir - # Skullstripping - skullstrip = Node(BET(frac = 0.1, functional = True, mask = True), - name = 'skullstrip') + # BET Node - Skullstripping data + skull_stripping_func = Node(BET(), name = 'skull_stripping_func') + skull_stripping_func.inputs.frac = 0.1 + skull_stripping_func.inputs.functional = True + skull_stripping_func.inputs.mask = True - # Smoothing - smooth = Node(IsotropicSmooth(fwhm = self.fwhm), # TODO : Previously set to 6 mm ? - name = 'smooth') + # IsotropicSmooth Node - Smoothing data + smoothing_func = Node(IsotropicSmooth(), name = 'smoothing_func') + smoothing_func.inputs.fwhm = self.fwhm # TODO : Previously set to 6 mm ? - # Function node get_subject_infos - get subject specific condition information - subject_infos = Node(Function( + # Function Node get_subject_infos - Get subject specific condition information + subject_information = Node(Function( function = self.get_session_infos, input_names = ['event_file'], - output_names = ['subject_info']), - name = 'subject_infos') + output_names = ['subject_info'] + ), name = 'subject_information') - # SpecifyModel - generates Model - specify_model = Node(SpecifyModel( - high_pass_filter_cutoff = 100, input_units = 'secs', time_repetition = self.tr), - name = 'specify_model') + # SpecifyModel Node - Generate run level model + specify_model = Node(SpecifyModel(), name = 'specify_model') + specify_model.inputs.high_pass_filter_cutoff = 100 + specify_model.inputs.input_units = 'secs' + specify_model.inputs.time_repetition = TaskInformation()['RepetitionTime'] - # Node contrasts to get contrasts + # Funcion Node get_contrasts - Get the list of contrasts contrasts = Node(Function( function = self.get_contrasts, input_names = [], - output_names = ['contrasts']), - name = 'contrasts') + output_names = ['contrasts'] + ), name = 'contrasts') - # Function node get_parameters_file - get parameters files + # Function Node get_parameters_file - Get files with movement parameters parameters = Node(Function( function = self.get_parameters_file, input_names = ['filepath', 'subject_id', 'run_id', 'working_dir'], @@ -259,21 +262,17 @@ def get_run_level_analysis(self): name = 'parameters') parameters.inputs.working_dir = self.directories.working_dir - # First temporal derivatives of the two regressors were also used, - # along with temporal filtering (60 s) of all the independent variable time-series. - # No motion parameter regressors used. - # Level1Design - generates a design matrix - l1_design = Node(Level1Design( - bases = {'dgamma':{'derivs' : True}}, - interscan_interval = self.tr, - model_serial_correlations = True), - name = 'l1_design') + # Level1Design Node - Generate files for run level computation + model_design = Node(Level1Design(), name = 'model_design') + model_design.inputs.bases = {'dgamma':{'derivs' : True}} + model_design.inputs.interscan_interval = TaskInformation()['RepetitionTime'] + model_design.inputs.model_serial_correlations = True - model_generation = Node(FEATModel(), - name = 'model_generation') + # FEATModel Node - Generate run level model + model_generation = Node(FEATModel(), name = 'model_generation') - model_estimate = Node(FILMGLS(), - name='model_estimate') + # FILMGLS Node - Estimate first level model + model_estimate = Node(FILMGLS(), name = 'model_estimate') # Function node remove_smoothed_files - remove output of the smooth node remove_smoothed_files = Node(Function( @@ -283,42 +282,45 @@ def get_run_level_analysis(self): remove_smoothed_files.inputs.working_dir = self.directories.working_dir # Create l1 analysis workflow and connect its nodes - l1_analysis = Workflow(base_dir = self.directories.working_dir, name = 'l1_analysis') - l1_analysis.connect([ - (infosource, selectfiles, [ + run_level_analysis = Workflow( + base_dir = self.directories.working_dir, + name = 'run_level_analysis' + ) + run_level_analysis.connect([ + (information_source, select_files, [ ('subject_id', 'subject_id'), ('run_id', 'run_id')]), - (selectfiles, subject_infos, [('event', 'event_file')]), - (selectfiles, parameters, [('param', 'filepath')]), - (infosource, parameters, [ + (select_files, subject_information, [('event', 'event_file')]), + (select_files, parameters, [('param', 'filepath')]), + (information_source, parameters, [ ('subject_id', 'subject_id'), ('run_id', 'run_id')]), - (selectfiles, skullstrip, [('func', 'in_file')]), - (skullstrip, smooth, [('out_file', 'in_file')]), + (select_files, skull_stripping_func, [('func', 'in_file')]), + (skull_stripping_func, smoothing_func, [('out_file', 'in_file')]), (parameters, specify_model, [('parameters_file', 'realignment_parameters')]), - (smooth, specify_model, [('out_file', 'functional_runs')]), - (subject_infos, specify_model, [('subject_info', 'subject_info')]), - (contrasts, l1_design, [('contrasts', 'contrasts')]), - (specify_model, l1_design, [('session_info', 'session_info')]), - (l1_design, model_generation, [ + (smoothing_func, specify_model, [('out_file', 'functional_runs')]), + (subject_information, specify_model, [('subject_info', 'subject_info')]), + (contrasts, model_design, [('contrasts', 'contrasts')]), + (specify_model, model_design, [('session_info', 'session_info')]), + (model_design, model_generation, [ ('ev_files', 'ev_files'), ('fsf_files', 'fsf_file')]), - (smooth, model_estimate, [('out_file', 'in_file')]), + (smoothing_func, model_estimate, [('out_file', 'in_file')]), (model_generation, model_estimate, [ ('con_file', 'tcon_file'), ('design_file', 'design_file')]), - (infosource, remove_smoothed_files, [ + (information_source, remove_smoothed_files, [ ('subject_id', 'subject_id'), ('run_id', 'run_id')]), (model_estimate, remove_smoothed_files, [('results_dir', '_')]), - (model_estimate, datasink, [('results_dir', 'l1_analysis.@results')]), + (model_estimate, datasink, [('results_dir', 'run_level_analysis.@results')]), (model_generation, datasink, [ - ('design_file', 'l1_analysis.@design_file'), - ('design_image', 'l1_analysis.@design_img')]), - (skullstrip, datasink, [('mask_file', 'l1_analysis.@skullstriped')]) + ('design_file', 'run_level_analysis.@design_file'), + ('design_image', 'run_level_analysis.@design_img')]), + (skull_stripping_func, datasink, [('mask_file', 'run_level_analysis.@skullstriped')]) ]) - return l1_analysis + return run_level_analysis def get_run_level_outputs(self): """ Return the names of the files the run level analysis is supposed to generate. """ @@ -357,7 +359,7 @@ def get_run_level_outputs(self): parameter_sets = product(*parameters.values()) template = join( self.directories.output_dir, - 'l1_analysis', '_run_id_{run_id}_subject_id_{subject_id}','{file}' + 'run_level_analysis', '_run_id_{run_id}_subject_id_{subject_id}','{file}' ) return_list = [template.format(**dict(zip(parameters.keys(), parameter_values)))\ for parameter_values in parameter_sets] @@ -376,7 +378,7 @@ def get_run_level_outputs(self): parameter_sets = product(*parameters.values()) template = join( self.directories.output_dir, - 'l1_analysis', '_run_id_{run_id}_subject_id_{subject_id}','{file}' + 'run_level_analysis', '_run_id_{run_id}_subject_id_{subject_id}','{file}' ) return_list += [template.format(**dict(zip(parameters.keys(), parameter_values)))\ @@ -389,13 +391,13 @@ def get_subject_level_analysis(self): Create the subject level analysis workflow. Returns: - - l2_analysis: nipype.WorkFlow + - subject_level_analysis : nipype.WorkFlow """ # Infosource Node - To iterate on subject and runs - infosource_sub_level = Node(IdentityInterface( + information_source = Node(IdentityInterface( fields = ['subject_id', 'contrast_id']), - name = 'infosource_sub_level') - infosource_sub_level.iterables = [ + name = 'information_source') + information_source.iterables = [ ('subject_id', self.subject_list), ('contrast_id', self.contrast_list) ] @@ -403,65 +405,64 @@ def get_subject_level_analysis(self): # Templates to select files node templates = { 'cope' : join(self.directories.output_dir, - 'l1_analysis', '_run_id_*_subject_id_{subject_id}', 'results', + 'run_level_analysis', '_run_id_*_subject_id_{subject_id}', 'results', 'cope{contrast_id}.nii.gz'), 'varcope' : join(self.directories.output_dir, - 'l1_analysis', '_run_id_*_subject_id_{subject_id}', 'results', + 'run_level_analysis', '_run_id_*_subject_id_{subject_id}', 'results', 'varcope{contrast_id}.nii.gz'), 'mask': join(self.directories.output_dir, - 'l1_analysis', '_run_id_*_subject_id_{subject_id}', + 'run_level_analysis', '_run_id_*_subject_id_{subject_id}', 'sub-{subject_id}_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') } - # SelectFiles node - to select necessary files - selectfiles_sub_level = Node(SelectFiles( - templates, base_directory = self.directories.results_dir), - name = 'selectfiles_sub_level') + # SelectFiles Node - to select necessary files + select_files = Node(SelectFiles(templates), name = 'select_files') + select_files.inputs.base_directory = self.directories.results_dir - # Datasink - save important files - datasink_sub_level = Node(DataSink( - base_directory = str(self.directories.output_dir) - ), - name = 'datasink_sub_level') + # DataSink Node - store the wanted results in the wanted directory + data_sink = Node(DataSink(), name = 'data_sink') + data_sink.inputs.base_directory = self.directories.output_dir + + # L2Model Node - Generate subject specific second level model + generate_model = Node(L2Model(), name = 'generate_model') + generate_model.inputs.num_copes = len(self.run_list) - # Generate design matrix - specify_model_sub_level = Node(L2Model( - num_copes = len(self.run_list)), - name = 'specify_model_sub_level') + # Merge Node - Merge copes files for each subject + merge_copes = Node(Merge(), name = 'merge_copes') + merge_copes.inputs.dimension = 't' - # Merge copes and varcopes files for each subject - merge_copes = Node(Merge(dimension = 't'), - name='merge_copes') - merge_varcopes = Node(Merge(dimension='t'), - name='merge_varcopes') + # Merge Node - Merge varcopes files for each subject + merge_varcopes = Node(Merge(), name = 'merge_varcopes') + merge_varcopes.inputs.dimension = 't' - flame = Node(FLAMEO(run_mode = 'flame1'), - name='flameo') + # FLAMEO Node - Estimate model + estimate_model = Node(FLAMEO(), name = 'estimate_model') + estimate_model.inputs.run_mode = 'flame1' # Second level (single-subject, mean of all four scans) analyses: Fixed effects analysis. - l2_analysis = Workflow( + subject_level_analysis = Workflow( base_dir = self.directories.working_dir, - name = 'l2_analysis') - l2_analysis.connect([ - (infosource_sub_level, selectfiles_sub_level, [ + name = 'subject_level_analysis') + subject_level_analysis.connect([ + (information_source, select_files, [ ('subject_id', 'subject_id'), ('contrast_id', 'contrast_id')]), - (selectfiles_sub_level, merge_copes, [('cope', 'in_files')]), - (selectfiles_sub_level, merge_varcopes, [('varcope', 'in_files')]), - (selectfiles_sub_level, flame, [('mask', 'mask_file')]), - (merge_copes, flame, [('merged_file', 'cope_file')]), - (merge_varcopes, flame, [('merged_file', 'var_cope_file')]), - (specify_model_sub_level, flame, [ + (select_files, merge_copes, [('cope', 'in_files')]), + (select_files, merge_varcopes, [('varcope', 'in_files')]), + (select_files, estimate_model, [('mask', 'mask_file')]), + (merge_copes, estimate_model, [('merged_file', 'cope_file')]), + (merge_varcopes, estimate_model, [('merged_file', 'var_cope_file')]), + (generate_model, estimate_model, [ ('design_mat', 'design_file'), ('design_con', 't_con_file'), ('design_grp', 'cov_split_file')]), - (flame, datasink_sub_level, [ - ('zstats', 'l2_analysis.@stats'), - ('tstats', 'l2_analysis.@tstats'), - ('copes', 'l2_analysis.@copes'), - ('var_copes', 'l2_analysis.@varcopes')])]) + (estimate_model, data_sink, [ + ('zstats', 'subject_level_analysis.@stats'), + ('tstats', 'subject_level_analysis.@tstats'), + ('copes', 'subject_level_analysis.@copes'), + ('var_copes', 'subject_level_analysis.@varcopes')])]) - return l2_analysis + return subject_level_analysis def get_subject_level_outputs(self): """ Return the names of the files the subject level analysis is supposed to generate. """ @@ -474,103 +475,120 @@ def get_subject_level_outputs(self): parameter_sets = product(*parameters.values()) template = join( self.directories.output_dir, - 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}','{file}' + 'subject_level_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}','{file}' ) return [template.format(**dict(zip(parameters.keys(), parameter_values)))\ for parameter_values in parameter_sets] - def get_subgroups_contrasts(copes, varcopes, subject_ids, participants_file): + def get_subgroups_contrasts(copes, varcopes, subject_list: list, participants_file: str): """ + Return the file list containing only the files belonging to subject in the wanted group. + Parameters : - - copes: original file list selected by selectfiles node - - varcopes: original file list selected by selectfiles node - - subject_ids: list of subject IDs that are analyzed - - participants_file: str, file containing participants characteristics + - copes: original file list selected by select_files node + - varcopes: original file list selected by select_files node + - subject_list: list of subject IDs that are analyzed + - participants_file: file containing participants characteristics - Returns : the file list containing only the files belonging to subject in the wanted group. + Returns : + - copes_equal_indifference : a subset of copes corresponding to subjects + in the equalIndifference group + - copes_equal_range : a subset of copes corresponding to subjects + in the equalRange group + - varcopes_equal_indifference : a subset of varcopes corresponding to subjects + in the equalIndifference group + - varcopes_equal_range : a subset of varcopes corresponding to subjects + in the equalRange group + - equal_indifference_ids : a list of subject ids in the equalIndifference group + - equal_range_ids : a list of subject ids in the equalRange group """ - equal_range_id = [] - equal_indifference_id = [] - subject_list = [f'sub-{str(s).zfill(3)}' for s in subject_ids] + subject_list_sub_ids = [] # ids as written in the participants file + equal_range_ids = [] # ids as 3-digit string + equal_indifference_ids = [] # ids as 3-digit string + equal_range_sub_ids = [] # ids as written in the participants file + equal_indifference_sub_ids = [] # ids as written in the participants file + + # Reading file containing participants IDs and groups with open(participants_file, 'rt') as file: next(file) # skip the header for line in file: info = line.strip().split() - if info[0] in subject_list and info[1] == 'equalIndifference': - equal_indifference_id.append(info[0][-3:]) - elif info[0] in subject_list and info[1] == 'equalRange': - equal_range_id.append(info[0][-3:]) - - copes_equal_indifference = [] - copes_equal_range = [] - copes_global = [] - varcopes_equal_indifference = [] - varcopes_equal_range = [] - varcopes_global = [] - - for file in copes: - sub_id = file.split('/') - if sub_id[-1][4:7] in equal_indifference_id: - copes_equal_indifference.append(file) - elif sub_id[-1][4:7] in equal_range_id: - copes_equal_range.append(file) - if sub_id[-1][4:7] in subject_ids: - copes_global.append(file) - - for file in varcopes: - sub_id = file.split('/') - if sub_id[-1][4:7] in equal_indifference_id: - varcopes_equal_indifference.append(file) - elif sub_id[-1][4:7] in equal_range_id: - varcopes_equal_range.append(file) - if sub_id[-1][4:7] in subject_ids: - varcopes_global.append(file) - - return copes_equal_indifference, copes_equal_range,\ - varcopes_equal_indifference, varcopes_equal_range,\ - equal_indifference_id, equal_range_id,\ - copes_global, varcopes_global - - def get_regressors(equal_range_ids, equal_indifference_ids, method, subject_list): + subject_id = info[0][-3:] + subject_group = info[1] + + # Check if the participant ID was selected and sort depending on group + if subject_id in subject_list: + subject_list_sub_ids.append(info[0]) + if subject_group == 'equalIndifference': + equal_indifference_ids.append(subject_id) + equal_indifference_sub_ids.append(info[0]) + elif subject_group == 'equalRange': + equal_range_ids.append(subject_id) + equal_range_sub_ids.append(info[0]) + + + # Return sorted selected copes and varcopes by group, and corresponding ids + return \ + [c for c in copes if any(i in c for i in equal_indifference_sub_ids)],\ + [c for c in copes if any(i in c for i in equal_range_sub_ids)],\ + [c for c in copes if any(i in c for i in subject_list_sub_ids)],\ + [v for v in varcopes if any(i in v for i in equal_indifference_sub_ids)],\ + [v for v in varcopes if any(i in v for i in equal_range_sub_ids)],\ + [v for v in varcopes if any(i in v for i in subject_list_sub_ids)],\ + equal_indifference_ids, equal_range_ids + + def get_one_sample_t_test_regressors(subject_list: list) -> dict: """ - Create dictionary of regressors for group analysis. + Create dictionary of regressors for one sample t-test group analysis. Parameters: - - equal_range_ids: list of str, ids of subjects in equal range group - - equal_indifference_ids: list of str, ids of subjects in equal indifference group - - method: one of 'equalRange', 'equalIndifference' or 'groupComp' - - subject_list: list of str, ids of subject for which to do the analysis + - subject_list: ids of subject in the group for which to do the analysis Returns: - - regressors: dict, dictionary of regressors used to - distinguish groups in FSL group analysis + - dict containing named lists of regressors. """ - if method == 'equalRange': - regressors = dict(group_mean = [1 for i in range(len(equal_range_ids))]) - elif method == 'equalIndifference': - regressors = dict(group_mean = [1 for i in range(len(equal_indifference_ids))]) - elif method == 'groupComp': - equal_range_reg = [ - 1 for i in range(len(equal_range_ids) + len(equal_indifference_ids)) - ] - equal_indifference_reg = [ - 0 for i in range(len(equal_range_ids) + len(equal_indifference_ids)) - ] - for index, sub_id in enumerate(subject_list): - if sub_id in equal_indifference_ids: - equal_indifference_reg[index] = 1 - equal_range_reg[index] = 0 + return dict(group_mean = [1 for _ in subject_list]) - regressors = dict( - equalRange = equal_range_reg, - equalIndifference = equal_indifference_reg - ) + def get_two_sample_t_test_regressors( + equal_range_ids: list, + equal_indifference_ids: list, + subject_list: list, + ) -> dict: + """ + Create dictionary of regressors for two sample t-test group analysis. + + Parameters: + - equal_range_ids: ids of subjects in equal range group + - equal_indifference_ids: ids of subjects in equal indifference group + - subject_list: ids of subject for which to do the analysis - return regressors + Returns: + - regressors, dict: containing named lists of regressors. + - groups, list: group identifiers to distinguish groups in FSL analysis. + """ + + # Create 2 lists containing n_sub values which are + # * 1 if the participant is on the group + # * 0 otherwise + equal_range_regressors = [1 if i in equal_range_ids else 0 for i in subject_list] + equal_indifference_regressors = [ + 1 if i in equal_indifference_ids else 0 for i in subject_list + ] + + # Create regressors output : a dict with the two list + regressors = dict( + equalRange = equal_range_regressors, + equalIndifference = equal_indifference_regressors + ) + + # Create groups outputs : a list with 1 for equalRange subjects and 2 for equalIndifference + groups = [1 if i == 1 else 2 for i in equal_range_regressors] + + return regressors, groups def get_group_level_analysis(self): """ @@ -591,19 +609,16 @@ def get_group_level_analysis_sub_workflow(self, method): - method: one of 'equalRange', 'equalIndifference' or 'groupComp' Returns: - - l3_analysis: nipype.WorkFlow + - group_level_analysis: nipype.WorkFlow """ - # Compute the number of participants used to do the analysis - nb_subjects = len(self.subject_list) - - # Infosource Node - To iterate on subject and runs - infosource_group_level = Node(IdentityInterface( + # Infosource Node - iterate over the contrasts generated by the subject level analysis + information_source = Node(IdentityInterface( fields = ['contrast_ids', 'subject_list'], subject_list = self.subject_list), - name = 'infosource_group_level') - infosource_group_level.iterables = [('contrast_id', self.contrast_list)] + name = 'information_source') + information_source.iterables = [('contrast_id', self.contrast_list)] - # Templates to select files node + # SelectFiles Node - select necessary files templates = { 'cope' : join(self.directories.output_dir, 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', @@ -616,126 +631,159 @@ def get_group_level_analysis_sub_workflow(self, method): 'l1_analysis', '_run_id_*_subject_id_{subject_id}', 'sub-{subject_id}_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') } - - # SelectFiles node - to select necessary files - selectfiles_group_level = Node(SelectFiles( - templates, base_directory = self.directories.results_dir), - name = 'selectfiles_group_level') - - datasink_group_level = Node(DataSink( - base_directory = self.directories.output_dir + select_files = Node(SelectFiles(templates), name = 'select_files') + select_files.inputs.base_directory = self.directories.results_dir + + # Datasink Node - save important files + data_sink = Node(DataSink(), name = 'data_sink') + data_sink.inputs.base_directory = self.directories.output_dir + + # Merge Node - Merge cope files + merge_copes = Node(Merge(), name = 'merge_copes') + merge_copes.inputs.dimension = 't' + + # Merge Node - Merge cope files + merge_varcopes = Node(Merge(), name = 'merge_varcopes') + merge_varcopes.inputs.dimension = 't' + + # Function Node get_one_sample_t_test_regressors + # Get regressors in the equalRange and equalIndifference method case + regressors_one_sample = Node( + Function( + function = self.get_one_sample_t_test_regressors, + input_names = ['subject_list'], + output_names = ['regressors'] ), - name='datasink_group_level') - - merge_copes_group_level = Node(Merge(dimension = 't'), - name = 'merge_copes_group_level') - merge_varcopes_group_level = Node(Merge(dimension = 't'), - name = 'merge_varcopes_group_level') + name = 'regressors_one_sample', + ) + + # Function Node get_two_sample_t_test_regressors + # Get regressors in the groupComp method case + regressors_two_sample = Node( + Function( + function = self.get_two_sample_t_test_regressors, + input_names = [ + 'equal_range_ids', + 'equal_indifference_ids', + 'subject_list', + ], + output_names = ['regressors', 'groups'] + ), + name = 'regressors_two_sample', + ) + regressors_two_sample.inputs.subject_list = self.subject_list - subgroups_contrasts = Node(Function( + # Function Node get_subgroups_contrasts - Get the contrast files for each subgroup + get_contrasts = Node(Function( function = self.get_subgroups_contrasts, input_names = ['copes', 'varcopes', 'subject_list', 'participants_file'], output_names = [ - 'copes_equalIndifference', 'copes_equalRange', - 'varcopes_equalIndifference', 'varcopes_equalRange', - 'equalIndifference_id', 'equalRange_id', - 'copes_global', 'varcopes_global'] + 'copes_equal_indifference', + 'copes_equal_range', + 'copes_global', + 'varcopes_equal_indifference', + 'varcopes_equal_range', + 'varcopes_global', + 'equal_indifference_id', + 'equal_range_id' + ] ), - name = 'subgroups_contrasts') + name = 'get_contrasts') - specifymodel_group_level = Node(MultipleRegressDesign(), - name = 'specifymodel_group_level') + # MultipleRegressDesign Node - Specify model + specify_model = Node(MultipleRegressDesign(), name = 'specify_model') - flame_group_level = Node(FLAMEO(run_mode = 'flame1'), - name='flame_group_level') + # FLAMEO Node - Estimate model + estimate_model = Node(FLAMEO(), name = 'estimate_model') + estimate_model.inputs.run_mode = 'flame1' - regressors = Node(Function( - function = self.get_regressors, - input_names = ['equalRange_id', 'equalIndifference_id', 'method', 'subject_list'], - output_names = ['regressors']), - name = 'regressors') - regressors.inputs.method = method - regressors.inputs.subject_list = self.subject_list + # Randomise Node - + randomise = Node(Randomise(), name = 'randomise') + randomise.inputs.num_perm = 10000 + randomise.inputs.tfce = True + randomise.inputs.vox_p_values = True + randomise.inputs.c_thresh = 0.05 + randomise.inputs.tfce_E = 0.01 - randomise = Node(Randomise( - num_perm = 10000, tfce = True, vox_p_values = True, c_thresh = 0.05, tfce_E = 0.01), - name = 'randomise') + # Compute the number of participants used to do the analysis + nb_subjects = len(self.subject_list) - l3_analysis = Workflow( + # Declare the workflow + group_level_analysis = Workflow( base_dir = self.directories.working_dir, - name = f'l3_analysis_{method}_nsub_{nb_subjects}') - l3_analysis.connect([ - (infosource_group_level, selectfiles_group_level, [('contrast_id', 'contrast_id')]), - (infosource_group_level, subgroups_contrasts, [('subject_list', 'subject_ids')]), - (selectfiles_group_level, subgroups_contrasts, [ + name = f'group_level_analysis_{method}_nsub_{nb_subjects}') + group_level_analysis.connect([ + (information_source, select_files, [('contrast_id', 'contrast_id')]), + (information_source, get_contrasts, [('subject_list', 'subject_list')]), + (select_files, get_contrasts, [ ('cope', 'copes'), ('varcope', 'varcopes'), ('participants', 'participants_file')]), - (selectfiles_group_level, flame_group_level, [('mask', 'mask_file')]), - (selectfiles_group_level, randomise, [('mask', 'mask')]), - (subgroups_contrasts, regressors, [ - ('equalRange_id', 'equalRange_id'), - ('equalIndifference_id', 'equalIndifference_id')]), - (regressors, specifymodel_group_level, [('regressors', 'regressors')])]) + (select_files, estimate_model, [('mask', 'mask_file')]), + (select_files, randomise, [('mask', 'mask')]) + ]) if method in ('equalIndifference', 'equalRange'): - specifymodel_group_level.inputs.contrasts = [ + specify_model.inputs.contrasts = [ ['group_mean', 'T', ['group_mean'], [1]], ['group_mean_neg', 'T', ['group_mean'], [-1]] ] + group_level_analysis.connect([ + (regressors_one_sample, specify_model, [('regressors', 'regressors')]) + ]) + if method == 'equalIndifference': - l3_analysis.connect([ - (subgroups_contrasts, merge_copes_group_level, - [('copes_equalIndifference', 'in_files')] - ), - (subgroups_contrasts, merge_varcopes_group_level, - [('varcopes_equalIndifference', 'in_files')] - ) + group_level_analysis.connect([ + (get_contrasts, merge_copes, [('copes_equal_indifference', 'in_files')]), + (get_contrasts, merge_varcopes,[('varcopes_equal_indifference', 'in_files')]), + (get_contrasts, regressors_one_sample, [ + ('equal_indifference_id', 'subject_list') + ]) ]) + elif method == 'equalRange': - l3_analysis.connect([ - (subgroups_contrasts, merge_copes_group_level, - [('copes_equalRange', 'in_files')] - ), - (subgroups_contrasts, merge_varcopes_group_level, - [('varcopes_equalRange', 'in_files')] - ) + group_level_analysis.connect([ + (get_contrasts, merge_copes, [('copes_equal_range', 'in_files')]), + (get_contrasts, merge_varcopes, [('varcopes_equal_range', 'in_files')]), + (get_contrasts, regressors_one_sample, [('equal_range_id', 'equal_range_id')]) ]) elif method == 'groupComp': - specifymodel_group_level.inputs.contrasts = [ + specify_model.inputs.contrasts = [ ['equalRange_sup', 'T', ['equalRange', 'equalIndifference'], [1, -1]] ] - l3_analysis.connect([ - (subgroups_contrasts, merge_copes_group_level, - [('copes_global', 'in_files')] - ), - (subgroups_contrasts, merge_varcopes_group_level, - [('varcopes_global', 'in_files')] - ) - ]) + group_level_analysis.connect([ + (get_contrasts, merge_copes, [('copes_global', 'in_files')]), + (get_contrasts, merge_varcopes,[('varcopes_global', 'in_files')]), + (get_contrasts, regressors_two_sample, [ + ('equal_range_id', 'equal_range_id'), + ('equal_indifference_id', 'equal_indifference_id')]), + (regressors_two_sample, specify_model, [ + ('regressors', 'regressors'), + ('groups', 'groups')]) + ]) - l3_analysis.connect([ - (merge_copes_group_level, flame_group_level, [('merged_file', 'cope_file')]), - (merge_varcopes_group_level, flame_group_level, [('merged_file', 'var_cope_file')]), - (specifymodel_group_level, flame_group_level, [ + group_level_analysis.connect([ + (merge_copes, estimate_model, [('merged_file', 'cope_file')]), + (merge_varcopes, estimate_model, [('merged_file', 'var_cope_file')]), + (specify_model, estimate_model, [ ('design_mat', 'design_file'), ('design_con', 't_con_file'), ('design_grp', 'cov_split_file')]), - (merge_copes_group_level, randomise, [('merged_file', 'in_file')]), - (specifymodel_group_level, randomise, [ + (merge_copes, randomise, [('merged_file', 'in_file')]), + (specify_model, randomise, [ ('design_mat', 'design_mat'), ('design_con', 'tcon')]), - (randomise, datasink_group_level, [ - ('t_corrected_p_files', f'l3_analysis_{method}_nsub_{nb_subjects}.@tcorpfile'), - ('tstat_files', f'l3_analysis_{method}_nsub_{nb_subjects}.@tstat')]), - (flame_group_level, datasink_group_level, [ - ('zstats', f'l3_analysis_{method}_nsub_{nb_subjects}.@zstats'), - ('tstats', f'l3_analysis_{method}_nsub_{nb_subjects}.@tstats')]), + (randomise, data_sink, [ + ('t_corrected_p_files', f'group_level_analysis_{method}_nsub_{nb_subjects}.@tcorpfile'), + ('tstat_files', f'group_level_analysis_{method}_nsub_{nb_subjects}.@tstat')]), + (estimate_model, data_sink, [ + ('zstats', f'group_level_analysis_{method}_nsub_{nb_subjects}.@zstats'), + ('tstats', f'group_level_analysis_{method}_nsub_{nb_subjects}.@tstats')]), ]) - return l3_analysis + return group_level_analysis def get_group_level_outputs(self): """ Return all names for the files the group level analysis is supposed to generate. """ From 3d96719c84005b89975a63570fd2b2fd35d30811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 16 Nov 2023 15:20:16 +0100 Subject: [PATCH 18/33] [TEST] adding unit tests for T54A [skip ci] --- narps_open/pipelines/team_T54A.py | 111 ++++++++--------- .../utils/configuration/testing_config.toml | 4 +- tests/pipelines/__init__.py | 33 +++++ tests/pipelines/test_team_T54A.py | 117 ++++++++++++++++++ 4 files changed, 204 insertions(+), 61 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 44daea20..161eea24 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -34,7 +34,7 @@ def get_preprocessing(self): """ No preprocessing has been done by team T54A """ return None - def get_session_infos(event_file): + def get_subject_information(event_file): """ Create Bunchs for specifyModel. @@ -46,7 +46,7 @@ def get_session_infos(event_file): """ from nipype.interfaces.base import Bunch - condition_names = ['trial', 'gain', 'loss', 'difficulty', 'response'] + condition_names = ['trial', 'gain', 'loss', 'difficulty', 'response', 'missed'] onset = {} duration = {} amplitude = {} @@ -62,36 +62,29 @@ def get_session_infos(event_file): for line in file: info = line.strip().split() - # Creates list with onsets, duration and loss/gain - # for amplitude (FSL) - for condition in condition_names: - if info[5] != 'NoResp': - if condition == 'gain': - onset[condition].append(float(info[0])) - duration[condition].append(float(info[4])) - amplitude[condition].append(float(info[2])) - elif condition == 'loss': - onset[condition].append(float(info[0])) - duration[condition].append(float(info[4])) - amplitude[condition].append(float(info[3])) - elif condition == 'trial': - onset[condition].append(float(info[0])) - duration[condition].append(float(info[4])) - amplitude[condition].append(float(1)) - elif condition == 'difficulty': - onset[condition].append(float(info[0])) - duration[condition].append(float(info[4])) - amplitude[condition].append( - abs(0.5 * float(info[2]) - float(info[3])) - ) - elif condition == 'response': - onset[condition].append(float(info[0]) + float(info[4])) - duration[condition].append(float(0)) - amplitude[condition].append(float(1)) - else: - if condition=='missed': - onset[condition].append(float(info[0])) - duration[condition].append(float(0)) + + if info[5] != 'NoResp': + onset['trial'].append(float(info[0])) + duration['trial'].append(float(info[4])) + amplitude['trial'].append(float(1)) + onset['gain'].append(float(info[0])) + duration['gain'].append(float(info[4])) + amplitude['gain'].append(float(info[2])) + onset['loss'].append(float(info[0])) + duration['loss'].append(float(info[4])) + amplitude['loss'].append(float(info[3])) + onset['difficulty'].append(float(info[0])) + duration['difficulty'].append(float(info[4])) + amplitude['difficulty'].append( + abs(0.5 * float(info[2]) - float(info[3])) + ) + onset['response'].append(float(info[0]) + float(info[4])) + duration['response'].append(float(0)) + amplitude['response'].append(float(1)) + else: + onset['missed'].append(float(info[0])) + duration['missed'].append(float(0)) + amplitude['missed'].append(float(1)) return [ Bunch( @@ -145,7 +138,7 @@ def get_parameters_file(filepath, subject_id, run_id, working_dir): return parameters_file - def get_contrasts(): + def get_run_level_contrasts(): """ Create a list of tuples that represent contrasts. Each contrast is in the form : @@ -236,7 +229,7 @@ def get_run_level_analysis(self): # Function Node get_subject_infos - Get subject specific condition information subject_information = Node(Function( - function = self.get_session_infos, + function = self.get_subject_information, input_names = ['event_file'], output_names = ['subject_info'] ), name = 'subject_information') @@ -247,9 +240,9 @@ def get_run_level_analysis(self): specify_model.inputs.input_units = 'secs' specify_model.inputs.time_repetition = TaskInformation()['RepetitionTime'] - # Funcion Node get_contrasts - Get the list of contrasts + # Funcion Node get_run_level_contrasts - Get the list of contrasts contrasts = Node(Function( - function = self.get_contrasts, + function = self.get_run_level_contrasts, input_names = [], output_names = ['contrasts'] ), name = 'contrasts') @@ -313,11 +306,11 @@ def get_run_level_analysis(self): ('subject_id', 'subject_id'), ('run_id', 'run_id')]), (model_estimate, remove_smoothed_files, [('results_dir', '_')]), - (model_estimate, datasink, [('results_dir', 'run_level_analysis.@results')]), - (model_generation, datasink, [ + (model_estimate, data_sink, [('results_dir', 'run_level_analysis.@results')]), + (model_generation, data_sink, [ ('design_file', 'run_level_analysis.@design_file'), ('design_image', 'run_level_analysis.@design_img')]), - (skull_stripping_func, datasink, [('mask_file', 'run_level_analysis.@skullstriped')]) + (skull_stripping_func, data_sink, [('mask_file', 'run_level_analysis.@skullstriped')]) ]) return run_level_analysis @@ -807,7 +800,7 @@ def get_group_level_outputs(self): parameter_sets = product(*parameters.values()) template = join( self.directories.output_dir, - 'l3_analysis_{method}_nsub_{nb_subjects}', + 'group_level_analysis_{method}_nsub_{nb_subjects}', '_contrast_id_{contrast_id}', '{file}' ) @@ -825,7 +818,7 @@ def get_group_level_outputs(self): return_list += [join( self.directories.output_dir, - f'l3_analysis_groupComp_nsub_{len(self.subject_list)}', + f'group_level_analysis_groupComp_nsub_{len(self.subject_list)}', '_contrast_id_ploss', f'{file}') for file in files] return return_list @@ -835,41 +828,41 @@ def get_hypotheses_outputs(self): nb_sub = len(self.subject_list) files = [ - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', + join(f'group_level_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', + join(f'group_level_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', + join(f'group_level_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', + join(f'group_level_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_pgain', 'zstat1.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat2.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat2.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', + join(f'group_level_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat2.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', + join(f'group_level_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat2.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalIndifference_nsub_{nb_sub}', + join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', + join(f'group_level_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_equalRange_nsub_{nb_sub}', + join(f'group_level_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat1.nii.gz'), - join(f'l3_analysis_groupComp_nsub_{nb_sub}', + join(f'group_level_analysis_groupComp_nsub_{nb_sub}', '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), - join(f'l3_analysis_groupComp_nsub_{nb_sub}', + join(f'group_level_analysis_groupComp_nsub_{nb_sub}', '_contrast_id_ploss', 'zstat1.nii.gz') ] return [join(self.directories.output_dir, f) for f in files] diff --git a/narps_open/utils/configuration/testing_config.toml b/narps_open/utils/configuration/testing_config.toml index b1fb28ba..40733c5a 100644 --- a/narps_open/utils/configuration/testing_config.toml +++ b/narps_open/utils/configuration/testing_config.toml @@ -3,9 +3,9 @@ title = "Testing configuration for the NARPS open pipelines project" config_type = "testing" [directories] -dataset = "run/data/ds001734/" +dataset = "data/original/ds001734/" reproduced_results = "run/data/reproduced/" -narps_results = "run/data/results/" +narps_results = "data/results/" test_data = "tests/test_data/" test_runs = "run/" diff --git a/tests/pipelines/__init__.py b/tests/pipelines/__init__.py index e69de29b..9ede3dbc 100644 --- a/tests/pipelines/__init__.py +++ b/tests/pipelines/__init__.py @@ -0,0 +1,33 @@ +#!/usr/bin/python +# coding: utf-8 + +""" +Configuration for testing of the narps_open.pipelines modules. +""" + +from pytest import helpers + +@helpers.register +def mock_event_data(mocker): + """ Mocks the return of the open function with the contents of a fake event file """ + + fake_event_data = 'onset duration\tgain\tloss\tRT\tparticipant_response\n' + fake_event_data += '4.071\t4\t14\t6\t2.388\tweakly_accept\n' + fake_event_data += '11.834\t4\t34\t14\t2.289\tstrongly_accept\n' + fake_event_data += '19.535\t4\t38\t19\t0\tNoResp\n' + fake_event_data += '27.535\t4\t10\t15\t2.08\tstrongly_reject\n' + fake_event_data += '36.435\t4\t16\t17\t2.288\tweakly_reject\n' + + mocker.patch('builtins.open', mocker.mock_open(read_data = fake_event_data)) + +@helpers.register +def mock_participants_data(mocker): + """ Mocks the return of the open function with the contents of a fake participants file """ + + fake_participants_data = 'participant_id\tgroup\tgender\tage\n' + fake_participants_data += 'sub-001\tequalIndifference\tM\t24\n' + fake_participants_data += 'sub-002\tequalRange\tM\t25\n' + fake_participants_data += 'sub-003\tequalIndifference\tF\t27\n' + fake_participants_data += 'sub-004\tequalRange\tM\t25\n' + + mocker.patch('builtins.open', mocker.mock_open(read_data = fake_participants_data)) diff --git a/tests/pipelines/test_team_T54A.py b/tests/pipelines/test_team_T54A.py index 0c3c2640..7f16728b 100644 --- a/tests/pipelines/test_team_T54A.py +++ b/tests/pipelines/test_team_T54A.py @@ -12,7 +12,9 @@ """ from pytest import helpers, mark +from numpy import isclose from nipype import Workflow +from nipype.interfaces.base import Bunch from narps_open.pipelines.team_T54A import PipelineTeamT54A @@ -61,6 +63,121 @@ def test_outputs(): assert len(pipeline.get_group_level_outputs()) == 8*2*2 + 4 assert len(pipeline.get_hypotheses_outputs()) == 18 + @staticmethod + @mark.unit_test + def test_subject_information(mocker): + """ Test the get_subject_information method """ + + helpers.mock_event_data(mocker) + + information = PipelineTeamT54A.get_subject_information('fake_event_file_path')[0] + + assert isinstance(information, Bunch) + assert information.conditions == [ + 'trial', + 'gain', + 'loss', + 'difficulty', + 'response', + 'missed' + ] + + reference_amplitudes = [ + [1.0, 1.0, 1.0, 1.0], + [14.0, 34.0, 10.0, 16.0], + [6.0, 14.0, 15.0, 17.0], + [1.0, 3.0, 10.0, 9.0], + [1.0, 1.0, 1.0, 1.0], + [1.0] + ] + for reference_array, test_array in zip(reference_amplitudes, information.amplitudes): + assert isclose(reference_array, test_array).all() + + reference_durations = [ + [2.388, 2.289, 2.08, 2.288], + [2.388, 2.289, 2.08, 2.288], + [2.388, 2.289, 2.08, 2.288], + [2.388, 2.289, 2.08, 2.288], + [0.0, 0.0, 0.0, 0.0], + [0.0] + ] + for reference_array, test_array in zip(reference_durations, information.durations): + assert isclose(reference_array, test_array).all() + + reference_onsets = [ + [4.071, 11.834, 27.535, 36.435], + [4.071, 11.834, 27.535, 36.435], + [4.071, 11.834, 27.535, 36.435], + [4.071, 11.834, 27.535, 36.435], + [6.459, 14.123, 29.615, 38.723], + [19.535] + ] + for reference_array, test_array in zip(reference_onsets, information.onsets): + assert isclose(reference_array, test_array).all() + + @staticmethod + @mark.unit_test + def test_parameters_file(mocker): + """ Test the get_parameters_file method """ + + + + @staticmethod + @mark.unit_test + def test_run_level_contrasts(): + """ Test the get_run_level_contrasts method """ + + contrasts = PipelineTeamT54A.get_run_level_contrasts() + assert contrasts[0] == ('gain', 'T', ['trial', 'gain', 'loss'], [0, 1, 0]) + assert contrasts[1] == ('loss', 'T', ['trial', 'gain', 'loss'], [0, 0, 1]) + + @staticmethod + @mark.unit_test + def test_subgroups_contrasts(mocker): + """ Test the get_subgroups_contrasts method """ + + helpers.mock_participants_data(mocker) + + cei, cer, cg, vei, ver, vg, eii, eri = PipelineTeamT54A.get_subgroups_contrasts( + ['sub-001/_contrast_id_1/cope1.nii.gz', 'sub-001/_contrast_id_2/cope1.nii.gz', 'sub-002/_contrast_id_1/cope1.nii.gz', 'sub-002/_contrast_id_2/cope1.nii.gz', 'sub-003/_contrast_id_1/cope1.nii.gz', 'sub-003/_contrast_id_2/cope1.nii.gz', 'sub-004/_contrast_id_1/cope1.nii.gz', 'sub-004/_contrast_id_2/cope1.nii.gz'], # copes + ['sub-001/_contrast_id_1/varcope1.nii.gz', 'sub-001/_contrast_id_2/varcope1.nii.gz', 'sub-002/_contrast_id_1/varcope1.nii.gz', 'sub-002/_contrast_id_2/varcope1.nii.gz', 'sub-003/_contrast_id_1/varcope1.nii.gz', 'sub-003/_contrast_id_2/varcope1.nii.gz', 'sub-004/_contrast_id_1/varcope1.nii.gz', 'sub-004/_contrast_id_2/varcope1.nii.gz'], # varcopes + ['001', '002', '003', '004'], # subject_list + ['fake_participants_file_path'] # participants file + ) + + assert cei == ['sub-001/_contrast_id_1/cope1.nii.gz', 'sub-001/_contrast_id_2/cope1.nii.gz', 'sub-003/_contrast_id_1/cope1.nii.gz', 'sub-003/_contrast_id_2/cope1.nii.gz'] + assert cer == ['sub-002/_contrast_id_1/cope1.nii.gz', 'sub-002/_contrast_id_2/cope1.nii.gz', 'sub-004/_contrast_id_1/cope1.nii.gz', 'sub-004/_contrast_id_2/cope1.nii.gz'] + assert cg == ['sub-001/_contrast_id_1/cope1.nii.gz', 'sub-001/_contrast_id_2/cope1.nii.gz', 'sub-002/_contrast_id_1/cope1.nii.gz', 'sub-002/_contrast_id_2/cope1.nii.gz', 'sub-003/_contrast_id_1/cope1.nii.gz', 'sub-003/_contrast_id_2/cope1.nii.gz', 'sub-004/_contrast_id_1/cope1.nii.gz', 'sub-004/_contrast_id_2/cope1.nii.gz'] + assert vei == ['sub-001/_contrast_id_1/varcope1.nii.gz', 'sub-001/_contrast_id_2/varcope1.nii.gz', 'sub-003/_contrast_id_1/varcope1.nii.gz', 'sub-003/_contrast_id_2/varcope1.nii.gz'] + assert ver == ['sub-002/_contrast_id_1/varcope1.nii.gz', 'sub-002/_contrast_id_2/varcope1.nii.gz', 'sub-004/_contrast_id_1/varcope1.nii.gz', 'sub-004/_contrast_id_2/varcope1.nii.gz'] + assert vg == ['sub-001/_contrast_id_1/varcope1.nii.gz', 'sub-001/_contrast_id_2/varcope1.nii.gz', 'sub-002/_contrast_id_1/varcope1.nii.gz', 'sub-002/_contrast_id_2/varcope1.nii.gz', 'sub-003/_contrast_id_1/varcope1.nii.gz', 'sub-003/_contrast_id_2/varcope1.nii.gz', 'sub-004/_contrast_id_1/varcope1.nii.gz', 'sub-004/_contrast_id_2/varcope1.nii.gz'] + assert eii == ['001', '003'] + assert eri == ['002', '004'] + + @staticmethod + @mark.unit_test + def test_one_sample_t_test_regressors(): + """ Test the get_one_sample_t_test_regressors method """ + + regressors = PipelineTeamT54A.get_one_sample_t_test_regressors(['001', '002']) + assert regressors == {'group_mean': [1, 1]} + + @staticmethod + @mark.unit_test + def test_two_sample_t_test_regressors(): + """ Test the get_two_sample_t_test_regressors method """ + + regressors, groups = PipelineTeamT54A.get_two_sample_t_test_regressors( + ['001', '003'], # equalRange group + ['002', '004'], # equalIndifference group + ['001', '002', '003', '004'] # all subjects + ) + assert regressors == dict( + equalRange = [1, 0, 1, 0], + equalIndifference = [0, 1, 0, 1] + ) + assert groups == [1, 2, 1, 2] + @staticmethod @mark.pipeline_test def test_execution(): From ae6178ec01004004f3a0c2bfc608f0204082486a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Fri, 17 Nov 2023 13:35:32 +0100 Subject: [PATCH 19/33] Removed get_contrasts method to write run level contrasts as class attributes instead [skip ci] --- narps_open/pipelines/team_T54A.py | 32 +++++-------------------------- tests/pipelines/test_team_T54A.py | 16 +++++----------- 2 files changed, 10 insertions(+), 38 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 161eea24..a680f042 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -29,6 +29,10 @@ def __init__(self): self.fwhm = 4.0 self.team_id = 'T54A' self.contrast_list = ['1', '2'] + self.run_level_contrasts = [ + ('gain', 'T', ['trial', 'gain', 'loss'], [0, 1, 0]), + ('loss', 'T', ['trial', 'gain', 'loss'], [0, 0, 1]) + ] def get_preprocessing(self): """ No preprocessing has been done by team T54A """ @@ -138,25 +142,6 @@ def get_parameters_file(filepath, subject_id, run_id, working_dir): return parameters_file - def get_run_level_contrasts(): - """ - Create a list of tuples that represent contrasts. - Each contrast is in the form : - (Name,Stat,[list of condition names],[weights on those conditions]) - - Returns: - - contrasts: list of tuples, list of contrasts to analyze - """ - # List of condition names - conditions = ['trial', 'gain', 'loss'] - - # Create contrasts - gain = ('gain', 'T', conditions, [0, 1, 0]) - loss = ('loss', 'T', conditions, [0, 0, 1]) - - # Contrast list - return [gain, loss] - def remove_smoothed_files(_, subject_id, run_id, working_dir): """ This method is used in a Function node to fully remove @@ -240,13 +225,6 @@ def get_run_level_analysis(self): specify_model.inputs.input_units = 'secs' specify_model.inputs.time_repetition = TaskInformation()['RepetitionTime'] - # Funcion Node get_run_level_contrasts - Get the list of contrasts - contrasts = Node(Function( - function = self.get_run_level_contrasts, - input_names = [], - output_names = ['contrasts'] - ), name = 'contrasts') - # Function Node get_parameters_file - Get files with movement parameters parameters = Node(Function( function = self.get_parameters_file, @@ -260,6 +238,7 @@ def get_run_level_analysis(self): model_design.inputs.bases = {'dgamma':{'derivs' : True}} model_design.inputs.interscan_interval = TaskInformation()['RepetitionTime'] model_design.inputs.model_serial_correlations = True + model_design.inputs.contrasts = self.run_level_contrasts # FEATModel Node - Generate run level model model_generation = Node(FEATModel(), name = 'model_generation') @@ -293,7 +272,6 @@ def get_run_level_analysis(self): (parameters, specify_model, [('parameters_file', 'realignment_parameters')]), (smoothing_func, specify_model, [('out_file', 'functional_runs')]), (subject_information, specify_model, [('subject_info', 'subject_info')]), - (contrasts, model_design, [('contrasts', 'contrasts')]), (specify_model, model_design, [('session_info', 'session_info')]), (model_design, model_generation, [ ('ev_files', 'ev_files'), diff --git a/tests/pipelines/test_team_T54A.py b/tests/pipelines/test_team_T54A.py index 7f16728b..e8b1f3a3 100644 --- a/tests/pipelines/test_team_T54A.py +++ b/tests/pipelines/test_team_T54A.py @@ -31,6 +31,11 @@ def test_create(): # 1 - check the parameters assert pipeline.fwhm == 4.0 assert pipeline.team_id == 'T54A' + assert pipeline.contrast_list == ['1', '2'] + assert pipeline.run_level_contrasts == [ + ('gain', 'T', ['trial', 'gain', 'loss'], [0, 1, 0]), + ('loss', 'T', ['trial', 'gain', 'loss'], [0, 0, 1]) + ] # 2 - check workflows assert pipeline.get_preprocessing() is None @@ -120,17 +125,6 @@ def test_subject_information(mocker): def test_parameters_file(mocker): """ Test the get_parameters_file method """ - - - @staticmethod - @mark.unit_test - def test_run_level_contrasts(): - """ Test the get_run_level_contrasts method """ - - contrasts = PipelineTeamT54A.get_run_level_contrasts() - assert contrasts[0] == ('gain', 'T', ['trial', 'gain', 'loss'], [0, 1, 0]) - assert contrasts[1] == ('loss', 'T', ['trial', 'gain', 'loss'], [0, 0, 1]) - @staticmethod @mark.unit_test def test_subgroups_contrasts(mocker): From 57d215172ebb238ef864fafad6a630f3fc6374dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Mon, 20 Nov 2023 15:46:29 +0100 Subject: [PATCH 20/33] [REFAC] end of refactoring T54A [TEST] end of unit_tests for T54A --- narps_open/pipelines/team_T54A.py | 376 +++++++++------------ tests/pipelines/__init__.py | 33 -- tests/pipelines/test_team_T54A.py | 58 ++-- tests/test_data/pipelines/confounds.tsv | 4 + tests/test_data/pipelines/events.tsv | 6 + tests/test_data/pipelines/participants.tsv | 5 + 6 files changed, 213 insertions(+), 269 deletions(-) delete mode 100644 tests/pipelines/__init__.py create mode 100644 tests/test_data/pipelines/confounds.tsv create mode 100644 tests/test_data/pipelines/events.tsv create mode 100644 tests/test_data/pipelines/participants.tsv diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index a680f042..9cedd408 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -6,7 +6,7 @@ from os.path import join from itertools import product -from nipype import Workflow, Node +from nipype import Workflow, Node, MapNode from nipype.interfaces.utility import IdentityInterface, Function from nipype.interfaces.io import SelectFiles, DataSink from nipype.interfaces.fsl import ( @@ -17,6 +17,8 @@ from narps_open.pipelines import Pipeline from narps_open.data.task import TaskInformation +from narps_open.data.participants import get_group +from narps_open.core.common import remove_file, list_intersection, elements_in_string, clean_list # Setup FSL FSLCommand.set_default_output_type('NIFTI_GZ') @@ -105,7 +107,7 @@ def get_parameters_file(filepath, subject_id, run_id, working_dir): Create a tsv file with only desired parameters per subject per run. Parameters : - - filepath : path to subject parameters file (i.e. one per run) + - filepath : path to the subject parameters file (i.e. one per run) - subject_id : subject for whom the 1st level analysis is made - run_id: run for which the 1st level analysis is made - working_dir: str, name of the directory for intermediate results @@ -132,7 +134,7 @@ def get_parameters_file(filepath, subject_id, run_id, working_dir): retained_parameters = DataFrame(transpose(temp_list)) parameters_file = join(working_dir, 'parameters_file', - f'parameters_file_sub-{subject_id}_run{run_id}.tsv') + f'parameters_file_sub-{subject_id}_run-{run_id}.tsv') makedirs(join(working_dir, 'parameters_file'), exist_ok = True) @@ -142,30 +144,6 @@ def get_parameters_file(filepath, subject_id, run_id, working_dir): return parameters_file - def remove_smoothed_files(_, subject_id, run_id, working_dir): - """ - This method is used in a Function node to fully remove - the files generated by the smoothing node, once they aren't needed anymore. - - Parameters: - - _: Node input only used for triggering the Node - - subject_id: str, id of the subject from which to remove the file - - run_id: str, id of the run from which to remove the file - - working_dir: str, path to the working dir - """ - from shutil import rmtree - from os.path import join - - try: - rmtree(join( - working_dir, 'l1_analysis', - f'_run_id_{run_id}_subject_id_{subject_id}', 'smooth') - ) - except OSError as error: - print(error) - else: - print('The directory is deleted successfully') - def get_run_level_analysis(self): """ Create the run level analysis workflow. @@ -248,10 +226,9 @@ def get_run_level_analysis(self): # Function node remove_smoothed_files - remove output of the smooth node remove_smoothed_files = Node(Function( - function = self.remove_smoothed_files, - input_names = ['_', 'subject_id', 'run_id', 'working_dir']), - name = 'remove_smoothed_files') - remove_smoothed_files.inputs.working_dir = self.directories.working_dir + function = remove_file, + input_names = ['_', 'file_name']), + name = 'remove_smoothed_files', iterfield = 'file_name') # Create l1 analysis workflow and connect its nodes run_level_analysis = Workflow( @@ -280,16 +257,14 @@ def get_run_level_analysis(self): (model_generation, model_estimate, [ ('con_file', 'tcon_file'), ('design_file', 'design_file')]), - (information_source, remove_smoothed_files, [ - ('subject_id', 'subject_id'), - ('run_id', 'run_id')]), + (smoothing_func, remove_smoothed_files, [('out_file', 'file_name')]), (model_estimate, remove_smoothed_files, [('results_dir', '_')]), (model_estimate, data_sink, [('results_dir', 'run_level_analysis.@results')]), (model_generation, data_sink, [ ('design_file', 'run_level_analysis.@design_file'), ('design_image', 'run_level_analysis.@design_img')]), (skull_stripping_func, data_sink, [('mask_file', 'run_level_analysis.@skullstriped')]) - ]) + ]) return run_level_analysis @@ -452,65 +427,6 @@ def get_subject_level_outputs(self): return [template.format(**dict(zip(parameters.keys(), parameter_values)))\ for parameter_values in parameter_sets] - def get_subgroups_contrasts(copes, varcopes, subject_list: list, participants_file: str): - """ - Return the file list containing only the files belonging to subject in the wanted group. - - Parameters : - - copes: original file list selected by select_files node - - varcopes: original file list selected by select_files node - - subject_list: list of subject IDs that are analyzed - - participants_file: file containing participants characteristics - - Returns : - - copes_equal_indifference : a subset of copes corresponding to subjects - in the equalIndifference group - - copes_equal_range : a subset of copes corresponding to subjects - in the equalRange group - - varcopes_equal_indifference : a subset of varcopes corresponding to subjects - in the equalIndifference group - - varcopes_equal_range : a subset of varcopes corresponding to subjects - in the equalRange group - - equal_indifference_ids : a list of subject ids in the equalIndifference group - - equal_range_ids : a list of subject ids in the equalRange group - """ - - subject_list_sub_ids = [] # ids as written in the participants file - equal_range_ids = [] # ids as 3-digit string - equal_indifference_ids = [] # ids as 3-digit string - equal_range_sub_ids = [] # ids as written in the participants file - equal_indifference_sub_ids = [] # ids as written in the participants file - - # Reading file containing participants IDs and groups - with open(participants_file, 'rt') as file: - next(file) # skip the header - - for line in file: - info = line.strip().split() - subject_id = info[0][-3:] - subject_group = info[1] - - # Check if the participant ID was selected and sort depending on group - if subject_id in subject_list: - subject_list_sub_ids.append(info[0]) - if subject_group == 'equalIndifference': - equal_indifference_ids.append(subject_id) - equal_indifference_sub_ids.append(info[0]) - elif subject_group == 'equalRange': - equal_range_ids.append(subject_id) - equal_range_sub_ids.append(info[0]) - - - # Return sorted selected copes and varcopes by group, and corresponding ids - return \ - [c for c in copes if any(i in c for i in equal_indifference_sub_ids)],\ - [c for c in copes if any(i in c for i in equal_range_sub_ids)],\ - [c for c in copes if any(i in c for i in subject_list_sub_ids)],\ - [v for v in varcopes if any(i in v for i in equal_indifference_sub_ids)],\ - [v for v in varcopes if any(i in v for i in equal_range_sub_ids)],\ - [v for v in varcopes if any(i in v for i in subject_list_sub_ids)],\ - equal_indifference_ids, equal_range_ids - def get_one_sample_t_test_regressors(subject_list: list) -> dict: """ Create dictionary of regressors for one sample t-test group analysis. @@ -592,14 +508,14 @@ def get_group_level_analysis_sub_workflow(self, method): # SelectFiles Node - select necessary files templates = { 'cope' : join(self.directories.output_dir, - 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', + 'subject_level_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', 'cope1.nii.gz'), 'varcope' : join(self.directories.output_dir, - 'l2_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', + 'subject_level_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', 'varcope1.nii.gz'), 'participants' : join(self.directories.dataset_dir, 'participants.tsv'), 'mask': join(self.directories.output_dir, - 'l1_analysis', '_run_id_*_subject_id_{subject_id}', + 'run_level_analysis', '_run_id_*_subject_id_{subject_id}', 'sub-{subject_id}_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') } select_files = Node(SelectFiles(templates), name = 'select_files') @@ -617,50 +533,6 @@ def get_group_level_analysis_sub_workflow(self, method): merge_varcopes = Node(Merge(), name = 'merge_varcopes') merge_varcopes.inputs.dimension = 't' - # Function Node get_one_sample_t_test_regressors - # Get regressors in the equalRange and equalIndifference method case - regressors_one_sample = Node( - Function( - function = self.get_one_sample_t_test_regressors, - input_names = ['subject_list'], - output_names = ['regressors'] - ), - name = 'regressors_one_sample', - ) - - # Function Node get_two_sample_t_test_regressors - # Get regressors in the groupComp method case - regressors_two_sample = Node( - Function( - function = self.get_two_sample_t_test_regressors, - input_names = [ - 'equal_range_ids', - 'equal_indifference_ids', - 'subject_list', - ], - output_names = ['regressors', 'groups'] - ), - name = 'regressors_two_sample', - ) - regressors_two_sample.inputs.subject_list = self.subject_list - - # Function Node get_subgroups_contrasts - Get the contrast files for each subgroup - get_contrasts = Node(Function( - function = self.get_subgroups_contrasts, - input_names = ['copes', 'varcopes', 'subject_list', 'participants_file'], - output_names = [ - 'copes_equal_indifference', - 'copes_equal_range', - 'copes_global', - 'varcopes_equal_indifference', - 'varcopes_equal_range', - 'varcopes_global', - 'equal_indifference_id', - 'equal_range_id' - ] - ), - name = 'get_contrasts') - # MultipleRegressDesign Node - Specify model specify_model = Node(MultipleRegressDesign(), name = 'specify_model') @@ -685,74 +557,157 @@ def get_group_level_analysis_sub_workflow(self, method): name = f'group_level_analysis_{method}_nsub_{nb_subjects}') group_level_analysis.connect([ (information_source, select_files, [('contrast_id', 'contrast_id')]), - (information_source, get_contrasts, [('subject_list', 'subject_list')]), - (select_files, get_contrasts, [ - ('cope', 'copes'), - ('varcope', 'varcopes'), - ('participants', 'participants_file')]), (select_files, estimate_model, [('mask', 'mask_file')]), - (select_files, randomise, [('mask', 'mask')]) + (select_files, randomise, [('mask', 'mask')]), + (merge_copes, estimate_model, [('merged_file', 'cope_file')]), + (merge_varcopes, estimate_model, [('merged_file', 'var_cope_file')]), + (specify_model, estimate_model, [ + ('design_mat', 'design_file'), + ('design_con', 't_con_file'), + ('design_grp', 'cov_split_file') + ]), + (merge_copes, randomise, [('merged_file', 'in_file')]), + (specify_model, randomise, [ + ('design_mat', 'design_mat'), + ('design_con', 'tcon') + ]), + (randomise, data_sink, [ + ('t_corrected_p_files', f'group_level_analysis_{method}_nsub_{nb_subjects}.@tcorpfile'), + ('tstat_files', f'group_level_analysis_{method}_nsub_{nb_subjects}.@tstat') + ]), + (estimate_model, data_sink, [ + ('zstats', f'group_level_analysis_{method}_nsub_{nb_subjects}.@zstats'), + ('tstats', f'group_level_analysis_{method}_nsub_{nb_subjects}.@tstats') + ]) ]) if method in ('equalIndifference', 'equalRange'): + + # Setup a one sample t-test specify_model.inputs.contrasts = [ ['group_mean', 'T', ['group_mean'], [1]], ['group_mean_neg', 'T', ['group_mean'], [-1]] ] - group_level_analysis.connect([ - (regressors_one_sample, specify_model, [('regressors', 'regressors')]) - ]) + # Function Node get_group_subjects - Get subjects in the group and in the subject_list + get_group_subjects = Node(Function( + function = list_intersection, + input_names = ['list_1', 'list_2'], + output_names = ['out_list'] + ), + name = 'get_group_subjects' + ) + get_group_subjects.inputs.list_1 = get_group(method) + get_group_subjects.inputs.list_2 = self.subject_list + + # Function Node elements_in_string + # Get contrast of parameter estimates (cope) for these subjects + # Note : using a MapNode with elements_in_string requires using clean_list to remove + # None values from the out_list + get_copes = MapNode(Function( + function = elements_in_string, + input_names = ['input_str', 'elements'], + output_names = ['out_list'] + ), + name = 'get_copes', iterfield = 'input_str' + ) - if method == 'equalIndifference': - group_level_analysis.connect([ - (get_contrasts, merge_copes, [('copes_equal_indifference', 'in_files')]), - (get_contrasts, merge_varcopes,[('varcopes_equal_indifference', 'in_files')]), - (get_contrasts, regressors_one_sample, [ - ('equal_indifference_id', 'subject_list') - ]) - ]) + # Function Node elements_in_string + # Get variance of the estimated copes (varcope) for these subjects + # Note : using a MapNode with elements_in_string requires using clean_list to remove + # None values from the out_list + get_varcopes = MapNode(Function( + function = elements_in_string, + input_names = ['input_str', 'elements'], + output_names = ['out_list'] + ), + name = 'get_varcopes', iterfield = 'input_str' + ) - elif method == 'equalRange': - group_level_analysis.connect([ - (get_contrasts, merge_copes, [('copes_equal_range', 'in_files')]), - (get_contrasts, merge_varcopes, [('varcopes_equal_range', 'in_files')]), - (get_contrasts, regressors_one_sample, [('equal_range_id', 'equal_range_id')]) - ]) + # Function Node get_one_sample_t_test_regressors + # Get regressors in the equalRange and equalIndifference method case + regressors_one_sample = Node( + Function( + function = self.get_one_sample_t_test_regressors, + input_names = ['subject_list'], + output_names = ['regressors'] + ), + name = 'regressors_one_sample', + ) + + # Add missing connections + group_level_analysis.connect([ + (select_files, get_copes, [('cope', 'input_str')]), + (select_files, get_varcopes, [('varcope', 'input_str')]), + (get_group_subjects, get_copes, [('out_list', 'elements')]), + (get_group_subjects, get_varcopes, [('out_list', 'elements')]), + (get_copes, merge_copes, [(('out_list', clean_list), 'in_files')]), + (get_varcopes, merge_varcopes,[(('out_list', clean_list), 'in_files')]), + (get_group_subjects, regressors_one_sample, [('out_list', 'subject_list')]), + (regressors_one_sample, specify_model, [('regressors', 'regressors')]) + ]) elif method == 'groupComp': + + # Setup a two sample t-test specify_model.inputs.contrasts = [ ['equalRange_sup', 'T', ['equalRange', 'equalIndifference'], [1, -1]] ] + + # Function Node get_equal_range_subjects + # Get subjects in the equalRange group and in the subject_list + get_equal_range_subjects = Node(Function( + function = list_intersection, + input_names = ['list_1', 'list_2'], + output_names = ['out_list'] + ), + name = 'get_equal_range_subjects' + ) + get_equal_range_subjects.inputs.list_1 = get_group('equalRange') + get_equal_range_subjects.inputs.list_2 = self.subject_list + + # Function Node get_equal_indifference_subjects + # Get subjects in the equalIndifference group and in the subject_list + get_equal_indifference_subjects = Node(Function( + function = list_intersection, + input_names = ['list_1', 'list_2'], + output_names = ['out_list'] + ), + name = 'get_equal_indifference_subjects' + ) + get_equal_indifference_subjects.inputs.list_1 = get_group('equalIndifference') + get_equal_indifference_subjects.inputs.list_2 = self.subject_list + + # Function Node get_two_sample_t_test_regressors + # Get regressors in the groupComp method case + regressors_two_sample = Node( + Function( + function = self.get_two_sample_t_test_regressors, + input_names = [ + 'equal_range_ids', + 'equal_indifference_ids', + 'subject_list', + ], + output_names = ['regressors', 'groups'] + ), + name = 'regressors_two_sample', + ) + regressors_two_sample.inputs.subject_list = self.subject_list + + # Add missing connections group_level_analysis.connect([ - (get_contrasts, merge_copes, [('copes_global', 'in_files')]), - (get_contrasts, merge_varcopes,[('varcopes_global', 'in_files')]), - (get_contrasts, regressors_two_sample, [ - ('equal_range_id', 'equal_range_id'), - ('equal_indifference_id', 'equal_indifference_id')]), + (select_files, merge_copes, [('cope', 'in_files')]), + (select_files, merge_varcopes,[('varcope', 'in_files')]), + (get_equal_range_subjects, regressors_two_sample, [ + ('out_list', 'equal_range_id') + ]), + (get_equal_indifference_subjects, regressors_two_sample, [ + ('out_list', 'equal_indifference_id') + ]), (regressors_two_sample, specify_model, [ ('regressors', 'regressors'), ('groups', 'groups')]) - ]) - - group_level_analysis.connect([ - (merge_copes, estimate_model, [('merged_file', 'cope_file')]), - (merge_varcopes, estimate_model, [('merged_file', 'var_cope_file')]), - (specify_model, estimate_model, [ - ('design_mat', 'design_file'), - ('design_con', 't_con_file'), - ('design_grp', 'cov_split_file')]), - (merge_copes, randomise, [('merged_file', 'in_file')]), - (specify_model, randomise, [ - ('design_mat', 'design_mat'), - ('design_con', 'tcon')]), - (randomise, data_sink, [ - ('t_corrected_p_files', f'group_level_analysis_{method}_nsub_{nb_subjects}.@tcorpfile'), - ('tstat_files', f'group_level_analysis_{method}_nsub_{nb_subjects}.@tstat')]), - (estimate_model, data_sink, [ - ('zstats', f'group_level_analysis_{method}_nsub_{nb_subjects}.@zstats'), - ('tstats', f'group_level_analysis_{method}_nsub_{nb_subjects}.@tstats')]), - ]) + ]) return group_level_analysis @@ -761,7 +716,7 @@ def get_group_level_outputs(self): # Handle equalRange and equalIndifference parameters = { - 'contrast_id': ['ploss', 'pgain'], + 'contrast_id': self.contrast_list, 'method': ['equalRange', 'equalIndifference'], 'file': [ 'randomise_tfce_corrp_tstat1.nii.gz', @@ -797,7 +752,7 @@ def get_group_level_outputs(self): return_list += [join( self.directories.output_dir, f'group_level_analysis_groupComp_nsub_{len(self.subject_list)}', - '_contrast_id_ploss', f'{file}') for file in files] + '_contrast_id_2', f'{file}') for file in files] return return_list @@ -807,43 +762,40 @@ def get_hypotheses_outputs(self): nb_sub = len(self.subject_list) files = [ join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', - '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + '_contrast_id_1', 'randomise_tfce_corrp_tstat1.nii.gz'), join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', - '_contrast_id_pgain', 'zstat1.nii.gz'), + '_contrast_id_1', 'zstat1.nii.gz'), join(f'group_level_analysis_equalRange_nsub_{nb_sub}', - '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + '_contrast_id_1', 'randomise_tfce_corrp_tstat1.nii.gz'), join(f'group_level_analysis_equalRange_nsub_{nb_sub}', - '_contrast_id_pgain', 'zstat1.nii.gz'), + '_contrast_id_1', 'zstat1.nii.gz'), join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', - '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + '_contrast_id_1', 'randomise_tfce_corrp_tstat1.nii.gz'), join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', - '_contrast_id_pgain', 'zstat1.nii.gz'), + '_contrast_id_1', 'zstat1.nii.gz'), join(f'group_level_analysis_equalRange_nsub_{nb_sub}', - '_contrast_id_pgain', 'randomise_tfce_corrp_tstat1.nii.gz'), + '_contrast_id_1', 'randomise_tfce_corrp_tstat1.nii.gz'), join(f'group_level_analysis_equalRange_nsub_{nb_sub}', - '_contrast_id_pgain', 'zstat1.nii.gz'), + '_contrast_id_1', 'zstat1.nii.gz'), join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', - '_contrast_id_ploss', 'randomise_tfce_corrp_tstat2.nii.gz'), + '_contrast_id_2', 'randomise_tfce_corrp_tstat2.nii.gz'), join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', - '_contrast_id_ploss', 'zstat2.nii.gz'), + '_contrast_id_2', 'zstat2.nii.gz'), join(f'group_level_analysis_equalRange_nsub_{nb_sub}', - '_contrast_id_ploss', 'randomise_tfce_corrp_tstat2.nii.gz'), + '_contrast_id_2', 'randomise_tfce_corrp_tstat2.nii.gz'), join(f'group_level_analysis_equalRange_nsub_{nb_sub}', - '_contrast_id_ploss', 'zstat2.nii.gz'), + '_contrast_id_2', 'zstat2.nii.gz'), join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', - '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), + '_contrast_id_2', 'randomise_tfce_corrp_tstat1.nii.gz'), join(f'group_level_analysis_equalIndifference_nsub_{nb_sub}', - '_contrast_id_ploss', 'zstat1.nii.gz'), + '_contrast_id_2', 'zstat1.nii.gz'), join(f'group_level_analysis_equalRange_nsub_{nb_sub}', - '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), + '_contrast_id_2', 'randomise_tfce_corrp_tstat1.nii.gz'), join(f'group_level_analysis_equalRange_nsub_{nb_sub}', - '_contrast_id_ploss', 'zstat1.nii.gz'), + '_contrast_id_2', 'zstat1.nii.gz'), join(f'group_level_analysis_groupComp_nsub_{nb_sub}', - '_contrast_id_ploss', 'randomise_tfce_corrp_tstat1.nii.gz'), + '_contrast_id_2', 'randomise_tfce_corrp_tstat1.nii.gz'), join(f'group_level_analysis_groupComp_nsub_{nb_sub}', - '_contrast_id_ploss', 'zstat1.nii.gz') + '_contrast_id_2', 'zstat1.nii.gz') ] return [join(self.directories.output_dir, f) for f in files] - -##### TODO : what is this ? -#system('export PATH=$PATH:/local/egermani/ICA-AROMA') diff --git a/tests/pipelines/__init__.py b/tests/pipelines/__init__.py deleted file mode 100644 index 9ede3dbc..00000000 --- a/tests/pipelines/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/python -# coding: utf-8 - -""" -Configuration for testing of the narps_open.pipelines modules. -""" - -from pytest import helpers - -@helpers.register -def mock_event_data(mocker): - """ Mocks the return of the open function with the contents of a fake event file """ - - fake_event_data = 'onset duration\tgain\tloss\tRT\tparticipant_response\n' - fake_event_data += '4.071\t4\t14\t6\t2.388\tweakly_accept\n' - fake_event_data += '11.834\t4\t34\t14\t2.289\tstrongly_accept\n' - fake_event_data += '19.535\t4\t38\t19\t0\tNoResp\n' - fake_event_data += '27.535\t4\t10\t15\t2.08\tstrongly_reject\n' - fake_event_data += '36.435\t4\t16\t17\t2.288\tweakly_reject\n' - - mocker.patch('builtins.open', mocker.mock_open(read_data = fake_event_data)) - -@helpers.register -def mock_participants_data(mocker): - """ Mocks the return of the open function with the contents of a fake participants file """ - - fake_participants_data = 'participant_id\tgroup\tgender\tage\n' - fake_participants_data += 'sub-001\tequalIndifference\tM\t24\n' - fake_participants_data += 'sub-002\tequalRange\tM\t25\n' - fake_participants_data += 'sub-003\tequalIndifference\tF\t27\n' - fake_participants_data += 'sub-004\tequalRange\tM\t25\n' - - mocker.patch('builtins.open', mocker.mock_open(read_data = fake_participants_data)) diff --git a/tests/pipelines/test_team_T54A.py b/tests/pipelines/test_team_T54A.py index e8b1f3a3..b585d9b0 100644 --- a/tests/pipelines/test_team_T54A.py +++ b/tests/pipelines/test_team_T54A.py @@ -10,13 +10,28 @@ pytest -q test_team_T54A.py pytest -q test_team_T54A.py -k """ +from os import mkdir +from os.path import exists, join +from shutil import rmtree -from pytest import helpers, mark +from pytest import helpers, mark, fixture from numpy import isclose from nipype import Workflow from nipype.interfaces.base import Bunch from narps_open.pipelines.team_T54A import PipelineTeamT54A +from narps_open.utils.configuration import Configuration + +TEMPORARY_DIR = join(Configuration()['directories']['test_runs'], 'test_T54A') + +@fixture +def remove_test_dir(): + """ A fixture to remove temporary directory created by tests """ + + rmtree(TEMPORARY_DIR, ignore_errors = True) + mkdir(TEMPORARY_DIR) + yield # test runs here + rmtree(TEMPORARY_DIR, ignore_errors = True) class TestPipelinesTeamT54A: """ A class that contains all the unit tests for the PipelineTeamT54A class.""" @@ -70,12 +85,13 @@ def test_outputs(): @staticmethod @mark.unit_test - def test_subject_information(mocker): + def test_subject_information(): """ Test the get_subject_information method """ - helpers.mock_event_data(mocker) + event_file_path = join( + Configuration()['directories']['test_data'], 'pipelines', 'events.tsv') - information = PipelineTeamT54A.get_subject_information('fake_event_file_path')[0] + information = PipelineTeamT54A.get_subject_information(event_file_path)[0] assert isinstance(information, Bunch) assert information.conditions == [ @@ -122,31 +138,25 @@ def test_subject_information(mocker): @staticmethod @mark.unit_test - def test_parameters_file(mocker): + def test_parameters_file(remove_test_dir): """ Test the get_parameters_file method """ - @staticmethod - @mark.unit_test - def test_subgroups_contrasts(mocker): - """ Test the get_subgroups_contrasts method """ - - helpers.mock_participants_data(mocker) + confounds_file_path = join( + Configuration()['directories']['test_data'], 'pipelines', 'confounds.tsv') - cei, cer, cg, vei, ver, vg, eii, eri = PipelineTeamT54A.get_subgroups_contrasts( - ['sub-001/_contrast_id_1/cope1.nii.gz', 'sub-001/_contrast_id_2/cope1.nii.gz', 'sub-002/_contrast_id_1/cope1.nii.gz', 'sub-002/_contrast_id_2/cope1.nii.gz', 'sub-003/_contrast_id_1/cope1.nii.gz', 'sub-003/_contrast_id_2/cope1.nii.gz', 'sub-004/_contrast_id_1/cope1.nii.gz', 'sub-004/_contrast_id_2/cope1.nii.gz'], # copes - ['sub-001/_contrast_id_1/varcope1.nii.gz', 'sub-001/_contrast_id_2/varcope1.nii.gz', 'sub-002/_contrast_id_1/varcope1.nii.gz', 'sub-002/_contrast_id_2/varcope1.nii.gz', 'sub-003/_contrast_id_1/varcope1.nii.gz', 'sub-003/_contrast_id_2/varcope1.nii.gz', 'sub-004/_contrast_id_1/varcope1.nii.gz', 'sub-004/_contrast_id_2/varcope1.nii.gz'], # varcopes - ['001', '002', '003', '004'], # subject_list - ['fake_participants_file_path'] # participants file + PipelineTeamT54A.get_parameters_file( + confounds_file_path, + 'fake_subject_id', + 'fake_run_id', + TEMPORARY_DIR ) - assert cei == ['sub-001/_contrast_id_1/cope1.nii.gz', 'sub-001/_contrast_id_2/cope1.nii.gz', 'sub-003/_contrast_id_1/cope1.nii.gz', 'sub-003/_contrast_id_2/cope1.nii.gz'] - assert cer == ['sub-002/_contrast_id_1/cope1.nii.gz', 'sub-002/_contrast_id_2/cope1.nii.gz', 'sub-004/_contrast_id_1/cope1.nii.gz', 'sub-004/_contrast_id_2/cope1.nii.gz'] - assert cg == ['sub-001/_contrast_id_1/cope1.nii.gz', 'sub-001/_contrast_id_2/cope1.nii.gz', 'sub-002/_contrast_id_1/cope1.nii.gz', 'sub-002/_contrast_id_2/cope1.nii.gz', 'sub-003/_contrast_id_1/cope1.nii.gz', 'sub-003/_contrast_id_2/cope1.nii.gz', 'sub-004/_contrast_id_1/cope1.nii.gz', 'sub-004/_contrast_id_2/cope1.nii.gz'] - assert vei == ['sub-001/_contrast_id_1/varcope1.nii.gz', 'sub-001/_contrast_id_2/varcope1.nii.gz', 'sub-003/_contrast_id_1/varcope1.nii.gz', 'sub-003/_contrast_id_2/varcope1.nii.gz'] - assert ver == ['sub-002/_contrast_id_1/varcope1.nii.gz', 'sub-002/_contrast_id_2/varcope1.nii.gz', 'sub-004/_contrast_id_1/varcope1.nii.gz', 'sub-004/_contrast_id_2/varcope1.nii.gz'] - assert vg == ['sub-001/_contrast_id_1/varcope1.nii.gz', 'sub-001/_contrast_id_2/varcope1.nii.gz', 'sub-002/_contrast_id_1/varcope1.nii.gz', 'sub-002/_contrast_id_2/varcope1.nii.gz', 'sub-003/_contrast_id_1/varcope1.nii.gz', 'sub-003/_contrast_id_2/varcope1.nii.gz', 'sub-004/_contrast_id_1/varcope1.nii.gz', 'sub-004/_contrast_id_2/varcope1.nii.gz'] - assert eii == ['001', '003'] - assert eri == ['002', '004'] + # Check parameter file was created + assert exists(join( + TEMPORARY_DIR, + 'parameters_file', + 'parameters_file_sub-fake_subject_id_run-fake_run_id.tsv') + ) @staticmethod @mark.unit_test diff --git a/tests/test_data/pipelines/confounds.tsv b/tests/test_data/pipelines/confounds.tsv new file mode 100644 index 00000000..f49d4fea --- /dev/null +++ b/tests/test_data/pipelines/confounds.tsv @@ -0,0 +1,4 @@ +CSF WhiteMatter GlobalSignal stdDVARS non-stdDVARS vx-wisestdDVARS FramewiseDisplacement tCompCor00 tCompCor01 tCompCor02 tCompCor03 tCompCor04 tCompCor05 aCompCor00 aCompCor01 aCompCor02 aCompCor03 aCompCor04 aCompCor05 Cosine00 Cosine01 Cosine02 Cosine03 Cosine04 Cosine05 NonSteadyStateOutlier00 X Y Z RotX RotY RotZ +6551.281999999999 6476.4653 9874.576 n/a n/a n/a n/a 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 -0.0 0.0 +6484.7285 6473.4890000000005 9830.212 1.09046686 52.78273392 1.05943739 0.13527900930999998 0.0263099209 -0.0673065879 0.0934882554 -0.0079328884 0.0338007737 -0.011491083999999999 -0.042411347099999996 0.027736422900000002 0.0453303087 -0.07022609490000001 0.0963618709 -0.0200867957 0.0665186088 0.0665174038 0.0665153954 0.0665125838 0.0665089688 0.06650455059999999 0.0 -0.00996895 -0.0313444 -3.00931e-06 0.00132687 -0.000384193 -0.00016819 +6441.5337 6485.7256 9821.212 1.07520139 52.04382706 1.03821933 0.12437666391 -0.0404820317 0.034150583 0.13661184210000002 0.0745358691 -0.0054829985999999995 -0.0217322686 0.046214115199999996 0.005774624 -0.043909359800000006 -0.075619539 0.17546891539999998 -0.0345256763 0.0665153954 0.06650455059999999 0.06648647719999999 0.0664611772 0.0664286533 0.0663889091 0.0 -2.56954e-05 -0.00923735 0.0549667 0.000997278 -0.00019745 -0.000398988 diff --git a/tests/test_data/pipelines/events.tsv b/tests/test_data/pipelines/events.tsv new file mode 100644 index 00000000..4b8f04e6 --- /dev/null +++ b/tests/test_data/pipelines/events.tsv @@ -0,0 +1,6 @@ +onset duration gain loss RT participant_response +4.071 4 14 6 2.388 weakly_accept +11.834 4 34 14 2.289 strongly_accept +19.535 4 38 19 0 NoResp +27.535 4 10 15 2.08 strongly_reject +36.435 4 16 17 2.288 weakly_reject \ No newline at end of file diff --git a/tests/test_data/pipelines/participants.tsv b/tests/test_data/pipelines/participants.tsv new file mode 100644 index 00000000..312dbcde --- /dev/null +++ b/tests/test_data/pipelines/participants.tsv @@ -0,0 +1,5 @@ +participant_id group gender age +sub-001 equalIndifference M 24 +sub-002 equalRange M 25 +sub-003 equalIndifference F 27 +sub-004 equalRange M 25 \ No newline at end of file From d2f267ac5de89a9d2a0ae72b36baf7d5f8e455b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Mon, 20 Nov 2023 17:42:34 +0100 Subject: [PATCH 21/33] Dealing with subjects selection in SelectFiles Nodes --- narps_open/pipelines/team_T54A.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 9cedd408..774b0319 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -500,23 +500,22 @@ def get_group_level_analysis_sub_workflow(self, method): """ # Infosource Node - iterate over the contrasts generated by the subject level analysis information_source = Node(IdentityInterface( - fields = ['contrast_ids', 'subject_list'], - subject_list = self.subject_list), + fields = ['contrast_id']), name = 'information_source') information_source.iterables = [('contrast_id', self.contrast_list)] # SelectFiles Node - select necessary files templates = { 'cope' : join(self.directories.output_dir, - 'subject_level_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', + 'subject_level_analysis', '_contrast_id_{contrast_id}_subject_id_*', 'cope1.nii.gz'), 'varcope' : join(self.directories.output_dir, - 'subject_level_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}', + 'subject_level_analysis', '_contrast_id_{contrast_id}_subject_id_*', 'varcope1.nii.gz'), 'participants' : join(self.directories.dataset_dir, 'participants.tsv'), 'mask': join(self.directories.output_dir, - 'run_level_analysis', '_run_id_*_subject_id_{subject_id}', - 'sub-{subject_id}_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') + 'run_level_analysis', '_run_id_*_subject_id_*', + 'sub-*_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') } select_files = Node(SelectFiles(templates), name = 'select_files') select_files.inputs.base_directory = self.directories.results_dir From 73173fac7601c2e49ae0ab2ef33b8e4985ac0d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 21 Nov 2023 11:32:46 +0100 Subject: [PATCH 22/33] [REFAC] group level : better identify groups --- narps_open/pipelines/team_T54A.py | 64 ++++++++++++++++--------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 774b0319..5ab44c35 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -512,7 +512,6 @@ def get_group_level_analysis_sub_workflow(self, method): 'varcope' : join(self.directories.output_dir, 'subject_level_analysis', '_contrast_id_{contrast_id}_subject_id_*', 'varcope1.nii.gz'), - 'participants' : join(self.directories.dataset_dir, 'participants.tsv'), 'mask': join(self.directories.output_dir, 'run_level_analysis', '_run_id_*_subject_id_*', 'sub-*_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') @@ -524,6 +523,30 @@ def get_group_level_analysis_sub_workflow(self, method): data_sink = Node(DataSink(), name = 'data_sink') data_sink.inputs.base_directory = self.directories.output_dir + # Function Node elements_in_string + # Get contrast of parameter estimates (cope) for these subjects + # Note : using a MapNode with elements_in_string requires using clean_list to remove + # None values from the out_list + get_copes = MapNode(Function( + function = elements_in_string, + input_names = ['input_str', 'elements'], + output_names = ['out_list'] + ), + name = 'get_copes', iterfield = 'input_str' + ) + + # Function Node elements_in_string + # Get variance of the estimated copes (varcope) for these subjects + # Note : using a MapNode with elements_in_string requires using clean_list to remove + # None values from the out_list + get_varcopes = MapNode(Function( + function = elements_in_string, + input_names = ['input_str', 'elements'], + output_names = ['out_list'] + ), + name = 'get_varcopes', iterfield = 'input_str' + ) + # Merge Node - Merge cope files merge_copes = Node(Merge(), name = 'merge_copes') merge_copes.inputs.dimension = 't' @@ -556,6 +579,10 @@ def get_group_level_analysis_sub_workflow(self, method): name = f'group_level_analysis_{method}_nsub_{nb_subjects}') group_level_analysis.connect([ (information_source, select_files, [('contrast_id', 'contrast_id')]), + (select_files, get_copes, [('cope', 'input_str')]), + (select_files, get_varcopes, [('varcope', 'input_str')]), + (get_copes, merge_copes, [(('out_list', clean_list), 'in_files')]), + (get_varcopes, merge_varcopes,[(('out_list', clean_list), 'in_files')]), (select_files, estimate_model, [('mask', 'mask_file')]), (select_files, randomise, [('mask', 'mask')]), (merge_copes, estimate_model, [('merged_file', 'cope_file')]), @@ -599,30 +626,6 @@ def get_group_level_analysis_sub_workflow(self, method): get_group_subjects.inputs.list_1 = get_group(method) get_group_subjects.inputs.list_2 = self.subject_list - # Function Node elements_in_string - # Get contrast of parameter estimates (cope) for these subjects - # Note : using a MapNode with elements_in_string requires using clean_list to remove - # None values from the out_list - get_copes = MapNode(Function( - function = elements_in_string, - input_names = ['input_str', 'elements'], - output_names = ['out_list'] - ), - name = 'get_copes', iterfield = 'input_str' - ) - - # Function Node elements_in_string - # Get variance of the estimated copes (varcope) for these subjects - # Note : using a MapNode with elements_in_string requires using clean_list to remove - # None values from the out_list - get_varcopes = MapNode(Function( - function = elements_in_string, - input_names = ['input_str', 'elements'], - output_names = ['out_list'] - ), - name = 'get_varcopes', iterfield = 'input_str' - ) - # Function Node get_one_sample_t_test_regressors # Get regressors in the equalRange and equalIndifference method case regressors_one_sample = Node( @@ -636,18 +639,19 @@ def get_group_level_analysis_sub_workflow(self, method): # Add missing connections group_level_analysis.connect([ - (select_files, get_copes, [('cope', 'input_str')]), - (select_files, get_varcopes, [('varcope', 'input_str')]), (get_group_subjects, get_copes, [('out_list', 'elements')]), (get_group_subjects, get_varcopes, [('out_list', 'elements')]), - (get_copes, merge_copes, [(('out_list', clean_list), 'in_files')]), - (get_varcopes, merge_varcopes,[(('out_list', clean_list), 'in_files')]), (get_group_subjects, regressors_one_sample, [('out_list', 'subject_list')]), (regressors_one_sample, specify_model, [('regressors', 'regressors')]) ]) elif method == 'groupComp': + # Select copes and varcopes corresponding to the selected subjects + # Indeed the SelectFiles node asks for all (*) subjects available + get_copes.inputs.elements = self.subject_list + get_varcopes.inputs.elements = self.subject_list + # Setup a two sample t-test specify_model.inputs.contrasts = [ ['equalRange_sup', 'T', ['equalRange', 'equalIndifference'], [1, -1]] @@ -695,8 +699,6 @@ def get_group_level_analysis_sub_workflow(self, method): # Add missing connections group_level_analysis.connect([ - (select_files, merge_copes, [('cope', 'in_files')]), - (select_files, merge_varcopes,[('varcope', 'in_files')]), (get_equal_range_subjects, regressors_two_sample, [ ('out_list', 'equal_range_id') ]), From 50d329d7e10c63dde0cd6d8ac5feacf5f7f9ae35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Wed, 22 Nov 2023 11:42:57 +0100 Subject: [PATCH 23/33] [BUG] with participants ids [skip ci] --- narps_open/data/participants.py | 8 ++++---- narps_open/pipelines/team_T54A.py | 4 ++-- tests/data/test_participants.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/narps_open/data/participants.py b/narps_open/data/participants.py index 835e834f..b0d6213e 100644 --- a/narps_open/data/participants.py +++ b/narps_open/data/participants.py @@ -51,9 +51,9 @@ def get_participants_subset(nb_participants: int = 108) -> list: return get_all_participants()[0:nb_participants] def get_group(group_name: str) -> list: - """ Return a list containing all the participants inside the group_name group + """ Return a list containing all the participants inside the group_name group """ - Warning : the subject ids are return as written in the participants file (i.e.: 'sub-*') - """ participants = get_participants_information() - return participants.loc[participants['group'] == group_name]['participant_id'].values.tolist() + group = participants.loc[participants['group'] == group_name]['participant_id'].values.tolist() + + return [p.replace('sub-', '') for p in group] diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 5ab44c35..101133f0 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -700,10 +700,10 @@ def get_group_level_analysis_sub_workflow(self, method): # Add missing connections group_level_analysis.connect([ (get_equal_range_subjects, regressors_two_sample, [ - ('out_list', 'equal_range_id') + ('out_list', 'equal_range_ids') ]), (get_equal_indifference_subjects, regressors_two_sample, [ - ('out_list', 'equal_indifference_id') + ('out_list', 'equal_indifference_ids') ]), (regressors_two_sample, specify_model, [ ('regressors', 'regressors'), diff --git a/tests/data/test_participants.py b/tests/data/test_participants.py index f36f0a05..eaf313fb 100644 --- a/tests/data/test_participants.py +++ b/tests/data/test_participants.py @@ -112,5 +112,5 @@ def test_get_group(mock_participants_data): """ Test the get_group function """ assert part.get_group('') == [] - assert part.get_group('equalRange') == ['sub-002', 'sub-004'] - assert part.get_group('equalIndifference') == ['sub-001', 'sub-003'] + assert part.get_group('equalRange') == ['002', '004'] + assert part.get_group('equalIndifference') == ['001', '003'] From 2fd6542614ff4cd14177671246db584c94647026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 21 Dec 2023 13:36:44 +0100 Subject: [PATCH 24/33] Dummy commit to run CI --- narps_open/pipelines/team_T54A.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 101133f0..f743eff8 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -24,7 +24,7 @@ FSLCommand.set_default_output_type('NIFTI_GZ') class PipelineTeamT54A(Pipeline): - """ A class that defines the pipeline of team T54A. """ + """ A class that defines the pipeline of team T54A """ def __init__(self): super().__init__() From a73f3c3346eb589b43ebdaa756367d8fb7e45b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 11 Jan 2024 14:53:15 +0100 Subject: [PATCH 25/33] [DATALAD] change results url --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index d3eaeb2f..364a3345 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,6 +5,6 @@ datalad-url = https://github.com/OpenNeuroDatasets/ds001734.git [submodule "data/results"] path = data/results - url = https://gin.g-node.org/RemiGau/neurovault_narps_open_pipeline.git - datalad-url = https://gin.g-node.org/RemiGau/neurovault_narps_open_pipeline.git + url = https://gin.g-node.org/RemiGau/neurovault_narps_open_pipeline + datalad-url = https://gin.g-node.org/RemiGau/neurovault_narps_open_pipeline datalad-id = b7b70790-7b0c-40d3-976f-c7dd49df3b86 From 26599bdd439b774f449ef46588cd3e314c5dc71f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 18 Jan 2024 10:53:20 +0100 Subject: [PATCH 26/33] [BUG] session information --- narps_open/pipelines/team_T54A.py | 56 +++++++++++++++++-------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index f743eff8..56156d46 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -53,15 +53,15 @@ def get_subject_information(event_file): from nipype.interfaces.base import Bunch condition_names = ['trial', 'gain', 'loss', 'difficulty', 'response', 'missed'] - onset = {} - duration = {} + onsets = {} + durations = {} amplitude = {} for condition in condition_names: # Create dictionary items with empty lists - onset.update({condition : []}) - duration.update({condition : []}) - amplitude.update({condition : []}) + onsets.update({condition : []}) + durations.update({condition : []}) + amplitudes.update({condition : []}) with open(event_file, 'rt') as file: next(file) # skip the header @@ -70,34 +70,38 @@ def get_subject_information(event_file): info = line.strip().split() if info[5] != 'NoResp': - onset['trial'].append(float(info[0])) - duration['trial'].append(float(info[4])) - amplitude['trial'].append(float(1)) - onset['gain'].append(float(info[0])) - duration['gain'].append(float(info[4])) - amplitude['gain'].append(float(info[2])) - onset['loss'].append(float(info[0])) - duration['loss'].append(float(info[4])) - amplitude['loss'].append(float(info[3])) - onset['difficulty'].append(float(info[0])) - duration['difficulty'].append(float(info[4])) - amplitude['difficulty'].append( + onsets['trial'].append(float(info[0])) + durations['trial'].append(float(info[4])) + amplitudes['trial'].append(float(1)) + onsets['gain'].append(float(info[0])) + durations['gain'].append(float(info[4])) + amplitudes['gain'].append(float(info[2])) + onsets['loss'].append(float(info[0])) + durations['loss'].append(float(info[4])) + amplitudes['loss'].append(float(info[3])) + onsets['difficulty'].append(float(info[0])) + durations['difficulty'].append(float(info[4])) + amplitudes['difficulty'].append( abs(0.5 * float(info[2]) - float(info[3])) ) - onset['response'].append(float(info[0]) + float(info[4])) - duration['response'].append(float(0)) - amplitude['response'].append(float(1)) + onsets['response'].append(float(info[0]) + float(info[4])) + durations['response'].append(0.0) + amplitudes['response'].append(1.0) else: - onset['missed'].append(float(info[0])) - duration['missed'].append(float(0)) - amplitude['missed'].append(float(1)) + onsets['missed'].append(float(info[0])) + durations['missed'].append(0.0) + amplitudes['missed'].append(1.0) + + # Check if there where missed trials for this run + if not onsets['missed']: + condition_names.remove('missed') return [ Bunch( conditions = condition_names, - onsets = [onset[k] for k in condition_names], - durations = [duration[k] for k in condition_names], - amplitudes = [amplitude[k] for k in condition_names], + onsets = [onsets[k] for k in condition_names], + durations = [durations[k] for k in condition_names], + amplitudes = [amplitudes[k] for k in condition_names], regressor_names = None, regressors = None) ] From fcc95d42cefb41379d87a7045c9f0732f6943ea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 18 Jan 2024 10:58:39 +0100 Subject: [PATCH 27/33] [BUG] session information --- narps_open/pipelines/team_T54A.py | 8 +- tests/pipelines/test_team_T54A.py | 97 +++++++++++++++-------- tests/test_data/pipelines/events_resp.tsv | 5 ++ 3 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 tests/test_data/pipelines/events_resp.tsv diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 56156d46..66da848f 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -55,7 +55,7 @@ def get_subject_information(event_file): condition_names = ['trial', 'gain', 'loss', 'difficulty', 'response', 'missed'] onsets = {} durations = {} - amplitude = {} + amplitudes = {} for condition in condition_names: # Create dictionary items with empty lists @@ -72,7 +72,7 @@ def get_subject_information(event_file): if info[5] != 'NoResp': onsets['trial'].append(float(info[0])) durations['trial'].append(float(info[4])) - amplitudes['trial'].append(float(1)) + amplitudes['trial'].append(1.0) onsets['gain'].append(float(info[0])) durations['gain'].append(float(info[4])) amplitudes['gain'].append(float(info[2])) @@ -258,9 +258,7 @@ def get_run_level_analysis(self): ('ev_files', 'ev_files'), ('fsf_files', 'fsf_file')]), (smoothing_func, model_estimate, [('out_file', 'in_file')]), - (model_generation, model_estimate, [ - ('con_file', 'tcon_file'), - ('design_file', 'design_file')]), + (model_generation, model_estimate, [('design_file', 'design_file')]), (smoothing_func, remove_smoothed_files, [('out_file', 'file_name')]), (model_estimate, remove_smoothed_files, [('results_dir', '_')]), (model_estimate, data_sink, [('results_dir', 'run_level_analysis.@results')]), diff --git a/tests/pipelines/test_team_T54A.py b/tests/pipelines/test_team_T54A.py index b585d9b0..569580b1 100644 --- a/tests/pipelines/test_team_T54A.py +++ b/tests/pipelines/test_team_T54A.py @@ -33,6 +33,14 @@ def remove_test_dir(): yield # test runs here rmtree(TEMPORARY_DIR, ignore_errors = True) +def compare_float_2d_arrays(array_1, array_2): + """ Assert array_1 and array_2 are close enough """ + + assert len(array_1) == len(array_2) + for reference_array, test_array in zip(array_1, array_2): + assert len(reference_array) == len(test_array) + assert isclose(reference_array, test_array).all() + class TestPipelinesTeamT54A: """ A class that contains all the unit tests for the PipelineTeamT54A class.""" @@ -88,53 +96,72 @@ def test_outputs(): def test_subject_information(): """ Test the get_subject_information method """ - event_file_path = join( - Configuration()['directories']['test_data'], 'pipelines', 'events.tsv') + # Get test files + test_file = join(Configuration()['directories']['test_data'], 'pipelines', 'events.tsv') + test_file_2 = join(Configuration()['directories']['test_data'], + 'pipelines', 'events_resp.tsv') - information = PipelineTeamT54A.get_subject_information(event_file_path)[0] + # Prepare several scenarii + info_missed = PipelineTeamT54A.get_subject_information(test_file) + info_ok = PipelineTeamT54A.get_subject_information(test_file_2) - assert isinstance(information, Bunch) - assert information.conditions == [ - 'trial', - 'gain', - 'loss', - 'difficulty', - 'response', - 'missed' - ] - - reference_amplitudes = [ - [1.0, 1.0, 1.0, 1.0], - [14.0, 34.0, 10.0, 16.0], - [6.0, 14.0, 15.0, 17.0], - [1.0, 3.0, 10.0, 9.0], - [1.0, 1.0, 1.0, 1.0], - [1.0] - ] - for reference_array, test_array in zip(reference_amplitudes, information.amplitudes): - assert isclose(reference_array, test_array).all() - - reference_durations = [ + # Compare bunches to expected + bunch = info_missed[0] + assert isinstance(bunch, Bunch) + assert bunch.conditions == ['trial', 'gain', 'loss', 'difficulty', 'response', 'missed'] + compare_float_2d_arrays(bunch.onsets, [ + [4.071, 11.834, 27.535, 36.435], + [4.071, 11.834, 27.535, 36.435], + [4.071, 11.834, 27.535, 36.435], + [4.071, 11.834, 27.535, 36.435], + [6.459, 14.123, 29.615, 38.723], + [19.535] + ]) + compare_float_2d_arrays(bunch.durations, [ [2.388, 2.289, 2.08, 2.288], [2.388, 2.289, 2.08, 2.288], [2.388, 2.289, 2.08, 2.288], [2.388, 2.289, 2.08, 2.288], [0.0, 0.0, 0.0, 0.0], [0.0] - ] - for reference_array, test_array in zip(reference_durations, information.durations): - assert isclose(reference_array, test_array).all() - - reference_onsets = [ + ]) + compare_float_2d_arrays(bunch.amplitudes, [ + [1.0, 1.0, 1.0, 1.0], + [14.0, 34.0, 10.0, 16.0], + [6.0, 14.0, 15.0, 17.0], + [1.0, 3.0, 10.0, 9.0], + [1.0, 1.0, 1.0, 1.0], + [1.0] + ]) + assert bunch.regressor_names == None + assert bunch.regressors == None + + bunch = info_ok[0] + assert isinstance(bunch, Bunch) + assert bunch.conditions == ['trial', 'gain', 'loss', 'difficulty', 'response'] + compare_float_2d_arrays(bunch.onsets, [ [4.071, 11.834, 27.535, 36.435], [4.071, 11.834, 27.535, 36.435], [4.071, 11.834, 27.535, 36.435], [4.071, 11.834, 27.535, 36.435], - [6.459, 14.123, 29.615, 38.723], - [19.535] - ] - for reference_array, test_array in zip(reference_onsets, information.onsets): - assert isclose(reference_array, test_array).all() + [6.459, 14.123, 29.615, 38.723] + ]) + compare_float_2d_arrays(bunch.durations, [ + [2.388, 2.289, 2.08, 2.288], + [2.388, 2.289, 2.08, 2.288], + [2.388, 2.289, 2.08, 2.288], + [2.388, 2.289, 2.08, 2.288], + [0.0, 0.0, 0.0, 0.0] + ]) + compare_float_2d_arrays(bunch.amplitudes, [ + [1.0, 1.0, 1.0, 1.0], + [14.0, 34.0, 10.0, 16.0], + [6.0, 14.0, 15.0, 17.0], + [1.0, 3.0, 10.0, 9.0], + [1.0, 1.0, 1.0, 1.0] + ]) + assert bunch.regressor_names == None + assert bunch.regressors == None @staticmethod @mark.unit_test diff --git a/tests/test_data/pipelines/events_resp.tsv b/tests/test_data/pipelines/events_resp.tsv new file mode 100644 index 00000000..dd5ea1a5 --- /dev/null +++ b/tests/test_data/pipelines/events_resp.tsv @@ -0,0 +1,5 @@ +onset duration gain loss RT participant_response +4.071 4 14 6 2.388 weakly_accept +11.834 4 34 14 2.289 strongly_accept +27.535 4 10 15 2.08 strongly_reject +36.435 4 16 17 2.288 weakly_reject \ No newline at end of file From 33f125b3ffa3abd260e2a4c33f06262f51ee5ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 18 Jan 2024 14:49:01 +0100 Subject: [PATCH 28/33] Correct value for fractional intensity threshold --- narps_open/pipelines/team_T54A.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 66da848f..0310c6b2 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -186,7 +186,7 @@ def get_run_level_analysis(self): # BET Node - Skullstripping data skull_stripping_func = Node(BET(), name = 'skull_stripping_func') - skull_stripping_func.inputs.frac = 0.1 + skull_stripping_func.inputs.frac = 0.3 skull_stripping_func.inputs.functional = True skull_stripping_func.inputs.mask = True From e3f9ab347208e15c042dcfe66b62979da09a138a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 23 Jan 2024 10:00:22 +0100 Subject: [PATCH 29/33] Run level contrast file --- narps_open/pipelines/team_T54A.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 0310c6b2..ed103a2a 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -258,7 +258,9 @@ def get_run_level_analysis(self): ('ev_files', 'ev_files'), ('fsf_files', 'fsf_file')]), (smoothing_func, model_estimate, [('out_file', 'in_file')]), - (model_generation, model_estimate, [('design_file', 'design_file')]), + (model_generation, model_estimate, [ + ('con_file', 'tcon_file'), + ('design_file', 'design_file')]), (smoothing_func, remove_smoothed_files, [('out_file', 'file_name')]), (model_estimate, remove_smoothed_files, [('results_dir', '_')]), (model_estimate, data_sink, [('results_dir', 'run_level_analysis.@results')]), From 52f2491703e71577b7894bf0b628c1166e135306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Wed, 24 Jan 2024 10:06:08 +0100 Subject: [PATCH 30/33] Merge run level masks for FLAMEo --- narps_open/pipelines/team_T54A.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index ed103a2a..68539f28 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -7,17 +7,18 @@ from itertools import product from nipype import Workflow, Node, MapNode -from nipype.interfaces.utility import IdentityInterface, Function +from nipype.interfaces.utility import IdentityInterface, Function, Split from nipype.interfaces.io import SelectFiles, DataSink from nipype.interfaces.fsl import ( BET, IsotropicSmooth, Level1Design, FEATModel, L2Model, Merge, FLAMEO, FILMGLS, Randomise, MultipleRegressDesign, FSLCommand ) from nipype.algorithms.modelgen import SpecifyModel +from nipype.interfaces.fsl.maths import MultiImageMaths from narps_open.pipelines import Pipeline from narps_open.data.task import TaskInformation -from narps_open.data.participants import get_group +from narps_open.data.participants import get_group from narps_open.core.common import remove_file, list_intersection, elements_in_string, clean_list # Setup FSL @@ -120,7 +121,7 @@ def get_parameters_file(filepath, subject_id, run_id, working_dir): - parameters_file : paths to new files containing only desired parameters. """ from os import makedirs - from os.path import join, isdir + from os.path import join from pandas import read_csv, DataFrame from numpy import array, transpose @@ -360,7 +361,7 @@ def get_subject_level_analysis(self): 'varcope' : join(self.directories.output_dir, 'run_level_analysis', '_run_id_*_subject_id_{subject_id}', 'results', 'varcope{contrast_id}.nii.gz'), - 'mask': join(self.directories.output_dir, + 'masks': join(self.directories.output_dir, 'run_level_analysis', '_run_id_*_subject_id_{subject_id}', 'sub-{subject_id}_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') } @@ -385,6 +386,16 @@ def get_subject_level_analysis(self): merge_varcopes = Node(Merge(), name = 'merge_varcopes') merge_varcopes.inputs.dimension = 't' + # Split Node - Split mask list to serve them as inputs of the MultiImageMaths node. + split_masks = Node(Split(), name = 'split_masks') + split_masks.inputs.splits = [1, len(self.run_list) - 1] + split_masks.inputs.squeeze = True # Unfold one-element splits removing the list + + # MultiImageMaths Node - Create a subject mask by + # computing the intersection of all run masks. + mask_intersection = Node(MultiImageMaths(), name = 'mask_intersection') + mask_intersection.inputs.op_string = '-mul %s ' * (len(self.run_list) - 1) + # FLAMEO Node - Estimate model estimate_model = Node(FLAMEO(), name = 'estimate_model') estimate_model.inputs.run_mode = 'flame1' @@ -399,13 +410,17 @@ def get_subject_level_analysis(self): ('contrast_id', 'contrast_id')]), (select_files, merge_copes, [('cope', 'in_files')]), (select_files, merge_varcopes, [('varcope', 'in_files')]), - (select_files, estimate_model, [('mask', 'mask_file')]), + (select_files, split_masks, [('masks', 'inlist')]), + (split_masks, mask_intersection, [('out1', 'in_file')]), + (split_masks, mask_intersection, [('out2', 'operand_files')]), + (mask_intersection, estimate_model, [('out_file', 'mask_file')]), (merge_copes, estimate_model, [('merged_file', 'cope_file')]), (merge_varcopes, estimate_model, [('merged_file', 'var_cope_file')]), (generate_model, estimate_model, [ ('design_mat', 'design_file'), ('design_con', 't_con_file'), ('design_grp', 'cov_split_file')]), + (mask_intersection, data_sink, [('out_file', 'subject_level_analysis.@mask')]), (estimate_model, data_sink, [ ('zstats', 'subject_level_analysis.@stats'), ('tstats', 'subject_level_analysis.@tstats'), From 23898e1ab01bd511efa2651c14475578c41f8787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Fri, 26 Jan 2024 17:08:58 +0100 Subject: [PATCH 31/33] T54A conditional removing files [skip ci] --- narps_open/pipelines/team_T54A.py | 33 ++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 68539f28..04476625 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -16,10 +16,12 @@ from nipype.algorithms.modelgen import SpecifyModel from nipype.interfaces.fsl.maths import MultiImageMaths +from narps_open.utils.configuration import Configuration from narps_open.pipelines import Pipeline from narps_open.data.task import TaskInformation from narps_open.data.participants import get_group -from narps_open.core.common import remove_file, list_intersection, elements_in_string, clean_list +from narps_open.core.common import list_intersection, elements_in_string, clean_list +from narps_open.core.interfaces import InterfaceFactory # Setup FSL FSLCommand.set_default_output_type('NIFTI_GZ') @@ -229,12 +231,6 @@ def get_run_level_analysis(self): # FILMGLS Node - Estimate first level model model_estimate = Node(FILMGLS(), name = 'model_estimate') - # Function node remove_smoothed_files - remove output of the smooth node - remove_smoothed_files = Node(Function( - function = remove_file, - input_names = ['_', 'file_name']), - name = 'remove_smoothed_files', iterfield = 'file_name') - # Create l1 analysis workflow and connect its nodes run_level_analysis = Workflow( base_dir = self.directories.working_dir, @@ -262,8 +258,6 @@ def get_run_level_analysis(self): (model_generation, model_estimate, [ ('con_file', 'tcon_file'), ('design_file', 'design_file')]), - (smoothing_func, remove_smoothed_files, [('out_file', 'file_name')]), - (model_estimate, remove_smoothed_files, [('results_dir', '_')]), (model_estimate, data_sink, [('results_dir', 'run_level_analysis.@results')]), (model_generation, data_sink, [ ('design_file', 'run_level_analysis.@design_file'), @@ -271,6 +265,27 @@ def get_run_level_analysis(self): (skull_stripping_func, data_sink, [('mask_file', 'run_level_analysis.@skullstriped')]) ]) + # Remove large files, if requested + if Configuration()['pipelines']['remove_unused_data']: + + # Remove Node - Remove skullstriped func files once they are no longer needed + remove_skullstrip = Node( + InterfaceFactory.create('remove_parent_directory'), + name = 'remove_skullstrip') + + # Remove Node - Remove smoothed files once they are no longer needed + remove_smooth = Node( + InterfaceFactory.create('remove_parent_directory'), + name = 'remove_smooth') + + # Add connections + run_level_analysis.connect([ + (data_sink, remove_skullstrip, [('out_file', '_')]), + (skull_stripping_func, remove_skullstrip, [('out_file', 'file_name')]), + (model_estimate, remove_smooth, [('results_dir', '_')]), + (smoothing_func, remove_smooth, [('out_file', 'file_name')]) + ]) + return run_level_analysis def get_run_level_outputs(self): From 9ed7ddd3eea18789a3563f251352198cc46709c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 30 Jan 2024 10:10:19 +0100 Subject: [PATCH 32/33] [SRC][TEST] level outputs & mask intersection for group level analysis --- narps_open/pipelines/team_T54A.py | 51 +++++++++++++------------------ tests/pipelines/test_team_T54A.py | 8 ++--- 2 files changed, 25 insertions(+), 34 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 04476625..87700ca4 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -295,30 +295,6 @@ def get_run_level_outputs(self): 'run_id' : self.run_list, 'subject_id' : self.subject_list, 'file' : [ - join('results', 'dof'), - join('results', 'logfile'), - join('results', 'pe10.nii.gz'), - join('results', 'pe11.nii.gz'), - join('results', 'pe12.nii.gz'), - join('results', 'pe13.nii.gz'), - join('results', 'pe14.nii.gz'), - join('results', 'pe15.nii.gz'), - join('results', 'pe16.nii.gz'), - join('results', 'pe17.nii.gz'), - join('results', 'pe1.nii.gz'), - join('results', 'pe2.nii.gz'), - join('results', 'pe3.nii.gz'), - join('results', 'pe4.nii.gz'), - join('results', 'pe5.nii.gz'), - join('results', 'pe6.nii.gz'), - join('results', 'pe7.nii.gz'), - join('results', 'pe8.nii.gz'), - join('results', 'pe9.nii.gz'), - join('results', 'res4d.nii.gz'), - join('results', 'sigmasquareds.nii.gz'), - join('results', 'threshac1.nii.gz'), - 'run0.mat', - 'run0.png', 'sub-{subject_id}_task-MGT_run-{run_id}_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz' ] } @@ -450,7 +426,9 @@ def get_subject_level_outputs(self): parameters = { 'contrast_id' : self.contrast_list, 'subject_id' : self.subject_list, - 'file' : ['cope1.nii.gz', 'tstat1.nii.gz', 'varcope1.nii.gz', 'zstat1.nii.gz'] + 'file' : ['cope1.nii.gz', 'tstat1.nii.gz', 'varcope1.nii.gz', 'zstat1.nii.gz', + 'sub-{subject_id}_task-MGT_run-01_bold_space-MNI152NLin2009cAsym_preproc_brain_mask_maths.nii.gz' + ] } parameter_sets = product(*parameters.values()) template = join( @@ -546,9 +524,9 @@ def get_group_level_analysis_sub_workflow(self, method): 'varcope' : join(self.directories.output_dir, 'subject_level_analysis', '_contrast_id_{contrast_id}_subject_id_*', 'varcope1.nii.gz'), - 'mask': join(self.directories.output_dir, - 'run_level_analysis', '_run_id_*_subject_id_*', - 'sub-*_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') + 'masks': join(self.directories.output_dir, + 'subject_level_analysis', '_contrast_id_1_subject_id_*', + 'sub-*_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc_brain_mask_maths.nii.gz') } select_files = Node(SelectFiles(templates), name = 'select_files') select_files.inputs.base_directory = self.directories.results_dir @@ -589,6 +567,16 @@ def get_group_level_analysis_sub_workflow(self, method): merge_varcopes = Node(Merge(), name = 'merge_varcopes') merge_varcopes.inputs.dimension = 't' + # Split Node - Split mask list to serve them as inputs of the MultiImageMaths node. + split_masks = Node(Split(), name = 'split_masks') + split_masks.inputs.splits = [1, len(self.subject_list) - 1] + split_masks.inputs.squeeze = True # Unfold one-element splits removing the list + + # MultiImageMaths Node - Create a subject mask by + # computing the intersection of all run masks. + mask_intersection = Node(MultiImageMaths(), name = 'mask_intersection') + mask_intersection.inputs.op_string = '-mul %s ' * (len(self.subject_list) - 1) + # MultipleRegressDesign Node - Specify model specify_model = Node(MultipleRegressDesign(), name = 'specify_model') @@ -617,8 +605,11 @@ def get_group_level_analysis_sub_workflow(self, method): (select_files, get_varcopes, [('varcope', 'input_str')]), (get_copes, merge_copes, [(('out_list', clean_list), 'in_files')]), (get_varcopes, merge_varcopes,[(('out_list', clean_list), 'in_files')]), - (select_files, estimate_model, [('mask', 'mask_file')]), - (select_files, randomise, [('mask', 'mask')]), + (select_files, split_masks, [('masks', 'inlist')]), + (split_masks, mask_intersection, [('out1', 'in_file')]), + (split_masks, mask_intersection, [('out2', 'operand_files')]), + (mask_intersection, estimate_model, [('out_file', 'mask_file')]), + (mask_intersection, randomise, [('out_file', 'mask')]), (merge_copes, estimate_model, [('merged_file', 'cope_file')]), (merge_varcopes, estimate_model, [('merged_file', 'var_cope_file')]), (specify_model, estimate_model, [ diff --git a/tests/pipelines/test_team_T54A.py b/tests/pipelines/test_team_T54A.py index 569580b1..7dfe4989 100644 --- a/tests/pipelines/test_team_T54A.py +++ b/tests/pipelines/test_team_T54A.py @@ -78,16 +78,16 @@ def test_outputs(): # 1 - 1 subject outputs pipeline.subject_list = ['001'] assert len(pipeline.get_preprocessing_outputs()) == 0 - assert len(pipeline.get_run_level_outputs()) == 33*4*1 - assert len(pipeline.get_subject_level_outputs()) == 4*2*1 + assert len(pipeline.get_run_level_outputs()) == 9*4*1 + assert len(pipeline.get_subject_level_outputs()) == 5*2*1 assert len(pipeline.get_group_level_outputs()) == 8*2*2 + 4 assert len(pipeline.get_hypotheses_outputs()) == 18 # 2 - 4 subjects outputs pipeline.subject_list = ['001', '002', '003', '004'] assert len(pipeline.get_preprocessing_outputs()) == 0 - assert len(pipeline.get_run_level_outputs()) == 33*4*4 - assert len(pipeline.get_subject_level_outputs()) == 4*2*4 + assert len(pipeline.get_run_level_outputs()) == 9*4*4 + assert len(pipeline.get_subject_level_outputs()) == 5*2*4 assert len(pipeline.get_group_level_outputs()) == 8*2*2 + 4 assert len(pipeline.get_hypotheses_outputs()) == 18 From 2aacae4e47367813067461968942770caedbb1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Wed, 31 Jan 2024 14:06:59 +0100 Subject: [PATCH 33/33] helper for testing pipeline outputs [skip ci] --- narps_open/pipelines/team_T54A.py | 53 +++++++++++++++---------------- setup.py | 1 + tests/conftest.py | 26 +++++++++++++++ tests/pipelines/test_team_T54A.py | 12 ++----- tests/test_conftest.py | 45 +++++++++++++++++++++++++- 5 files changed, 98 insertions(+), 39 deletions(-) diff --git a/narps_open/pipelines/team_T54A.py b/narps_open/pipelines/team_T54A.py index 87700ca4..ae34b848 100644 --- a/narps_open/pipelines/team_T54A.py +++ b/narps_open/pipelines/team_T54A.py @@ -291,38 +291,33 @@ def get_run_level_analysis(self): def get_run_level_outputs(self): """ Return the names of the files the run level analysis is supposed to generate. """ + return_list = [] + output_dir = join(self.directories.output_dir, 'run_level_analysis', + '_run_id_{run_id}_subject_id_{subject_id}') + + # Handle results dir parameters = { 'run_id' : self.run_list, 'subject_id' : self.subject_list, - 'file' : [ - 'sub-{subject_id}_task-MGT_run-{run_id}_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz' - ] + 'contrast_id' : self.contrast_list } parameter_sets = product(*parameters.values()) - template = join( - self.directories.output_dir, - 'run_level_analysis', '_run_id_{run_id}_subject_id_{subject_id}','{file}' - ) + templates = [ + join(output_dir, 'results', 'cope{contrast_id}.nii.gz'), + join(output_dir, 'results', 'tstat{contrast_id}.nii.gz'), + join(output_dir, 'results', 'varcope{contrast_id}.nii.gz'), + join(output_dir, 'results', 'zstat{contrast_id}.nii.gz') + ] return_list = [template.format(**dict(zip(parameters.keys(), parameter_values)))\ - for parameter_values in parameter_sets] + for parameter_values in parameter_sets for template in templates] + # Handle mask file parameters = { 'run_id' : self.run_list, 'subject_id' : self.subject_list, - 'contrast_id' : self.contrast_list, - 'file' : [ - join('results', 'cope{contrast_id}.nii.gz'), - join('results', 'tstat{contrast_id}.nii.gz'), - join('results', 'varcope{contrast_id}.nii.gz'), - join('results', 'zstat{contrast_id}.nii.gz'), - ] } parameter_sets = product(*parameters.values()) - template = join( - self.directories.output_dir, - 'run_level_analysis', '_run_id_{run_id}_subject_id_{subject_id}','{file}' - ) - + template = join(output_dir, 'sub-{subject_id}_task-MGT_run-{run_id}_bold_space-MNI152NLin2009cAsym_preproc_brain_mask.nii.gz') return_list += [template.format(**dict(zip(parameters.keys(), parameter_values)))\ for parameter_values in parameter_sets] @@ -426,18 +421,20 @@ def get_subject_level_outputs(self): parameters = { 'contrast_id' : self.contrast_list, 'subject_id' : self.subject_list, - 'file' : ['cope1.nii.gz', 'tstat1.nii.gz', 'varcope1.nii.gz', 'zstat1.nii.gz', - 'sub-{subject_id}_task-MGT_run-01_bold_space-MNI152NLin2009cAsym_preproc_brain_mask_maths.nii.gz' - ] } parameter_sets = product(*parameters.values()) - template = join( - self.directories.output_dir, - 'subject_level_analysis', '_contrast_id_{contrast_id}_subject_id_{subject_id}','{file}' - ) + output_dir = join(self.directories.output_dir, 'subject_level_analysis', + '_contrast_id_{contrast_id}_subject_id_{subject_id}') + templates = [ + join(output_dir, 'cope1.nii.gz'), + join(output_dir, 'tstat1.nii.gz'), + join(output_dir, 'varcope1.nii.gz'), + join(output_dir, 'zstat1.nii.gz'), + join(output_dir, 'sub-{subject_id}_task-MGT_run-01_bold_space-MNI152NLin2009cAsym_preproc_brain_mask_maths.nii.gz') + ] return [template.format(**dict(zip(parameters.keys(), parameter_values)))\ - for parameter_values in parameter_sets] + for parameter_values in parameter_sets for template in templates] def get_one_sample_t_test_regressors(subject_list: list) -> dict: """ diff --git a/setup.py b/setup.py index 91a2d63a..185c8418 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ ] extras_require = { 'tests': [ + 'pathvalidate', 'pylint', 'pytest', 'pytest-cov', diff --git a/tests/conftest.py b/tests/conftest.py index e01e4a00..d46df024 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,9 @@ from shutil import rmtree from pytest import helpers +from pathvalidate import is_valid_filepath +from narps_open.pipelines import Pipeline from narps_open.runner import PipelineRunner from narps_open.utils import get_subject_id from narps_open.utils.correlation import get_correlation_coefficient @@ -21,6 +23,30 @@ # Init configuration, to ensure it is in testing mode Configuration(config_type='testing') +@helpers.register +def test_pipeline_outputs(pipeline: Pipeline, number_of_outputs: list): + """ Test the outputs of a Pipeline. + Arguments: + - pipeline, Pipeline: the pipeline to test + - number_of_outputs, list: a list containing the expected number of outputs for each + stage of the pipeline (preprocessing, run_level, subject_level, group_level, hypotheses) + + Return: True if the outputs are in sufficient number and each ones name is valid, + False otherwise. + """ + assert len(number_of_outputs) == 5 + for outputs, number in zip([ + pipeline.get_preprocessing_outputs(), + pipeline.get_run_level_outputs(), + pipeline.get_subject_level_outputs(), + pipeline.get_group_level_outputs(), + pipeline.get_hypotheses_outputs()], number_of_outputs): + + assert len(outputs) == number + for output in outputs: + assert is_valid_filepath(output, platform = 'auto') + assert not any(c in output for c in ['{', '}']) + @helpers.register def test_pipeline_execution( team_id: str, diff --git a/tests/pipelines/test_team_T54A.py b/tests/pipelines/test_team_T54A.py index 7dfe4989..05ecc003 100644 --- a/tests/pipelines/test_team_T54A.py +++ b/tests/pipelines/test_team_T54A.py @@ -77,19 +77,11 @@ def test_outputs(): pipeline = PipelineTeamT54A() # 1 - 1 subject outputs pipeline.subject_list = ['001'] - assert len(pipeline.get_preprocessing_outputs()) == 0 - assert len(pipeline.get_run_level_outputs()) == 9*4*1 - assert len(pipeline.get_subject_level_outputs()) == 5*2*1 - assert len(pipeline.get_group_level_outputs()) == 8*2*2 + 4 - assert len(pipeline.get_hypotheses_outputs()) == 18 + helpers.test_pipeline_outputs(pipeline, [0, 9*4*1, 5*2*1, 8*2*2 + 4, 18]) # 2 - 4 subjects outputs pipeline.subject_list = ['001', '002', '003', '004'] - assert len(pipeline.get_preprocessing_outputs()) == 0 - assert len(pipeline.get_run_level_outputs()) == 9*4*4 - assert len(pipeline.get_subject_level_outputs()) == 5*2*4 - assert len(pipeline.get_group_level_outputs()) == 8*2*2 + 4 - assert len(pipeline.get_hypotheses_outputs()) == 18 + helpers.test_pipeline_outputs(pipeline, [0, 9*4*4, 5*2*4, 8*2*2 + 4, 18]) @staticmethod @mark.unit_test diff --git a/tests/test_conftest.py b/tests/test_conftest.py index de7d72cb..658f5d89 100644 --- a/tests/test_conftest.py +++ b/tests/test_conftest.py @@ -17,7 +17,7 @@ from datetime import datetime -from pytest import mark, helpers, fixture +from pytest import mark, helpers, fixture, raises from nipype import Node, Workflow from nipype.interfaces.utility import Function @@ -239,6 +239,49 @@ def download(self): class TestConftest: """ A class that contains all the unit tests for the conftest module.""" + @staticmethod + @mark.unit_test + def test_test_outputs(set_test_directory): + """ Test the test_pipeline_outputs helper """ + + # Test pipeline + pipeline = MockupPipeline() + pipeline.subject_list = ['001', '002'] + + # Wrong length for nb_of_outputs + with raises(AssertionError): + helpers.test_pipeline_outputs(pipeline, [1,2,3]) + + # Wrong number of outputs + with raises(AssertionError): + helpers.test_pipeline_outputs(pipeline, [0, 2, 2, 20, 18]) + with raises(AssertionError): + helpers.test_pipeline_outputs(pipeline, [2, 0, 2, 20, 18]) + with raises(AssertionError): + helpers.test_pipeline_outputs(pipeline, [2, 2, 0, 20, 18]) + with raises(AssertionError): + helpers.test_pipeline_outputs(pipeline, [2, 2, 2, 0, 18]) + with raises(AssertionError): + helpers.test_pipeline_outputs(pipeline, [2, 2, 2, 20, 0]) + + # Right number of outputs + helpers.test_pipeline_outputs(pipeline, [2, 2, 2, 20, 18]) + + # Not a valid path name + pipeline.get_group_level_outputs = lambda : 'not_fo\rmatted' + with raises(AssertionError): + helpers.test_pipeline_outputs(pipeline, [2, 2, 2, 1, 18]) + + # Not a valid path name + pipeline.get_group_level_outputs = lambda : '{not_formatted' + with raises(AssertionError): + helpers.test_pipeline_outputs(pipeline, [2, 2, 2, 1, 18]) + + # Not a valid path name + pipeline.get_group_level_outputs = lambda : '{not_formatted' + with raises(AssertionError): + helpers.test_pipeline_outputs(pipeline, [2, 2, 2, 1, 18]) + @staticmethod @mark.unit_test def test_test_correlation_results(mocker):