Pylint and code factorisation

.github/workflows/code_quality.yml

@@ -46,7 +46,7 @@ jobs:
 
     - name: Analyse the code with pylint
       run: |
-        pylint --fail-under 8 --ignore-paths narps_open/pipelines/ narps_open > pylint_report_narps_open.txt
+        pylint --fail-under 8 narps_open > pylint_report_narps_open.txt
         pylint --fail-under 8 tests > pylint_report_tests.txt
 
     - name: Archive pylint results

narps_open/pipelines/team_08MQ.py

@@ -386,23 +386,24 @@ def get_preprocessing_outputs(self):
         parameters = {
             'subject_id': self.subject_list,
             'run_id': self.run_list,
-            'file': [
-                'components_file.txt',
-                'sub-{subject_id}_task-MGT_run-{run_id}_bold_brain_mcf.nii.gz.par',
-                'sub-{subject_id}_task-MGT_run-{run_id}_bold_brain_mcf_st_smooth_flirt_wtsimt.nii.gz',
-                'sub-{subject_id}_task-MGT_run-{run_id}_bold_brain_mask_flirt_wtsimt.nii.gz'
-            ]
         }
         parameter_sets = product(*parameters.values())
-        template = join(
+        output_dir = join(
             self.directories.output_dir,
             'preprocessing',
-            '_run_id_{run_id}_subject_id_{subject_id}',
-            '{file}'
-            )
+            '_run_id_{run_id}_subject_id_{subject_id}'
+        )
+        templates = [
+            join(output_dir, 'components_file.txt'),
+            join(output_dir, 'sub-{subject_id}_task-MGT_run-{run_id}_bold_brain_mcf.nii.gz.par'),
+            join(output_dir,
+                'sub-{subject_id}_task-MGT_run-{run_id}_bold_brain_mcf_st_smooth_flirt_wtsimt.nii.gz'),
+            join(output_dir,
+                'sub-{subject_id}_task-MGT_run-{run_id}_bold_brain_mask_flirt_wtsimt.nii.gz')
+        ]
 
         return [template.format(**dict(zip(parameters.keys(), parameter_values)))\
-            for parameter_values in parameter_sets]
+            for parameter_values in parameter_sets for template in templates]
 
     def get_subject_information(event_file):
         """
@@ -586,21 +587,20 @@ def get_run_level_outputs(self):
             'run_id' : self.run_list,
             'subject_id' : self.subject_list,
             'contrast_id' : self.contrast_list,
-            'file' : [
-                join('results', 'cope{contrast_id}.nii.gz'),
-                join('results', 'tstat{contrast_id}.nii.gz'),
-                join('results', 'varcope{contrast_id}.nii.gz'),
-                join('results', 'zstat{contrast_id}.nii.gz'),
-            ]
         }
         parameter_sets = product(*parameters.values())
-        template = join(
+        output_dir = join(
             self.directories.output_dir,
-            'run_level_analysis', '_run_id_{run_id}_subject_id_{subject_id}','{file}'
+            'run_level_analysis', '_run_id_{run_id}_subject_id_{subject_id}'
             )
-
+        templates = [
+            join(output_dir, 'results', 'cope{contrast_id}.nii.gz'),
+            join(output_dir, 'results', 'tstat{contrast_id}.nii.gz'),
+            join(output_dir, 'results', 'varcope{contrast_id}.nii.gz'),
+            join(output_dir, 'results', 'zstat{contrast_id}.nii.gz'),
+        ]
         return_list += [template.format(**dict(zip(parameters.keys(), parameter_values)))\
-            for parameter_values in parameter_sets]
+            for parameter_values in parameter_sets for template in templates]
 
         return return_list
 
@@ -1005,6 +1005,57 @@ def get_group_level_analysis_sub_workflow(self, method):
 
         return group_level_analysis
 
+    def get_group_level_outputs(self):
+        """ Return all names for the files the group level analysis is supposed to generate. """
+
+        # Handle equalRange and equalIndifference
+        parameters = {
+            'contrast_id': self.contrast_list,
+            'method': ['equalRange', 'equalIndifference'],
+            'file': [
+                'randomise_tfce_corrp_tstat1.nii.gz',
+                'randomise_tfce_corrp_tstat2.nii.gz',
+                'randomise_tstat1.nii.gz',
+                'randomise_tstat2.nii.gz',
+                'tstat1.nii.gz',
+                'tstat2.nii.gz',
+                'zstat1.nii.gz',
+                'zstat2.nii.gz'
+                ],
+            'nb_subjects' : [str(len(self.subject_list))]
+        }
+        parameter_sets = product(*parameters.values())
+        template = join(
+            self.directories.output_dir,
+            'group_level_analysis_{method}_nsub_{nb_subjects}',
+            '_contrast_id_{contrast_id}',
+            '{file}'
+            )
+
+        return_list = [template.format(**dict(zip(parameters.keys(), parameter_values)))\
+            for parameter_values in parameter_sets]
+
+        # Handle groupComp
+        parameters = {
+            'contrast_id': self.contrast_list,
+            'file': [
+                'randomise_tfce_corrp_tstat1.nii.gz',
+                'randomise_tstat1.nii.gz',
+                'zstat1.nii.gz',
+                'tstat1.nii.gz'
+            ]
+        }
+        parameter_sets = product(*parameters.values())
+        template = join(
+            self.directories.output_dir,
+            f'group_level_analysis_groupComp_nsub_{len(self.subject_list)}',
+            '_contrast_id_{contrast_id}', '{file}')
+
+        return_list += [template.format(**dict(zip(parameters.keys(), parameter_values)))\
+            for parameter_values in parameter_sets]
+
+        return return_list
+
     def get_hypotheses_outputs(self):
         """ Return the names of the files used by the team to answer the hypotheses of NARPS. """
 

narps_open/pipelines/team_0I4U_debug.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 from nipype.interfaces.spm import (Coregister, Smooth, OneSampleTTestDesign, EstimateModel, EstimateContrast, 
                                    Level1Design, TwoSampleTTestDesign, RealignUnwarp, FieldMap, NewSegment, 
                                    Normalize12, Reslice)

narps_open/pipelines/team_1KB2_debug.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 from nipype.interfaces.fsl import (BET, FAST, MCFLIRT, FLIRT, FNIRT, ApplyWarp, SUSAN, 
                                    Info, ImageMaths, IsotropicSmooth, Threshold, Level1Design, FEATModel, 
                                    L2Model, Merge, FLAMEO, ContrastMgr,Cluster,  FILMGLS, Randomise, MultipleRegressDesign)

narps_open/pipelines/team_43FJ_debug.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 from nipype.interfaces.fsl import (BET, FAST, MCFLIRT, FLIRT, FNIRT, ApplyWarp, SUSAN, MotionOutliers,
                                    Info, ImageMaths, IsotropicSmooth, Threshold, Level1Design, FEATModel, 
                                    L2Model, Merge, FLAMEO, ContrastMgr, FILMGLS, Randomise, MultipleRegressDesign)

narps_open/pipelines/team_4TQ6_wip.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 from nipype.interfaces.fsl import (BET, ICA_AROMA, FAST, MCFLIRT, FLIRT, FNIRT, ApplyWarp, SUSAN, 
                                    Info, ImageMaths, IsotropicSmooth, Threshold, Level1Design, FEATModel, 
                                    L2Model, Merge, FLAMEO, ContrastMgr,Cluster,  FILMGLS, Randomise, MultipleRegressDesign)

narps_open/pipelines/team_98BT.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 from nipype.interfaces.spm import (Coregister, Smooth, OneSampleTTestDesign, EstimateModel, EstimateContrast, 
                                    Level1Design, TwoSampleTTestDesign, RealignUnwarp, NewSegment, SliceTiming,
                                   DARTEL, DARTELNorm2MNI, FieldMap)

narps_open/pipelines/team_Q6O0.py

@@ -468,11 +468,14 @@ def get_subject_level_outputs(self):
             )
 
         # Formatting templates and returning it as a list of files
-        output_files = [contrast_map_template.format(**dict(zip(parameters.keys(), parameter_values)))\
+        output_files = [
+            contrast_map_template.format(**dict(zip(parameters.keys(), parameter_values)))\
             for parameter_values in parameter_sets]
-        output_files += [mat_file_template.format(**dict(zip(parameters.keys(), parameter_values)))\
+        output_files += [
+            mat_file_template.format(**dict(zip(parameters.keys(), parameter_values)))\
             for parameter_values in parameter_sets]
-        output_files += [spmt_file_template.format(**dict(zip(parameters.keys(), parameter_values)))\
+        output_files += [
+            spmt_file_template.format(**dict(zip(parameters.keys(), parameter_values)))\
             for parameter_values in parameter_sets]
 
         return output_files
@@ -698,23 +701,41 @@ def get_hypotheses_outputs(self):
         """ Return all hypotheses output file names. """
         nb_sub = len(self.subject_list)
         files = [
-            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'),
-            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_gain', 'spmT_0001.nii'),
-            join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'),
-            join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_gain', 'spmT_0001.nii'),
-            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'),
-            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_gain', 'spmT_0001.nii'),
-            join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'),
-            join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_gain', 'spmT_0001.nii'),
-            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_loss', '_threshold1', 'spmT_0002_thr.nii'),
-            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_loss', 'spmT_0002.nii'),
-            join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_loss', '_threshold1', 'spmT_0002_thr.nii'),
-            join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_loss', 'spmT_0002.nii'),
-            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_loss', '_threshold0', 'spmT_0001_thr.nii'),
-            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_loss', 'spmT_0001.nii'),
-            join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_loss', '_threshold0', 'spmT_0001_thr.nii'),
-            join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_loss', 'spmT_0001.nii'),
-            join(f'l2_analysis_groupComp_nsub_{nb_sub}', '_model_type_loss', '_threshold0', 'spmT_0001_thr.nii'),
-            join(f'l2_analysis_groupComp_nsub_{nb_sub}', '_model_type_loss', 'spmT_0001.nii')
+            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}',
+                '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'),
+            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}',
+                '_model_type_gain', 'spmT_0001.nii'),
+            join(f'l2_analysis_equalRange_nsub_{nb_sub}',
+                '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'),
+            join(f'l2_analysis_equalRange_nsub_{nb_sub}',
+                '_model_type_gain', 'spmT_0001.nii'),
+            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}',
+                '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'),
+            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}',
+                '_model_type_gain', 'spmT_0001.nii'),
+            join(f'l2_analysis_equalRange_nsub_{nb_sub}',
+                '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'),
+            join(f'l2_analysis_equalRange_nsub_{nb_sub}',
+                '_model_type_gain', 'spmT_0001.nii'),
+            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}',
+                '_model_type_loss', '_threshold1', 'spmT_0002_thr.nii'),
+            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}',
+                '_model_type_loss', 'spmT_0002.nii'),
+            join(f'l2_analysis_equalRange_nsub_{nb_sub}',
+                '_model_type_loss', '_threshold1', 'spmT_0002_thr.nii'),
+            join(f'l2_analysis_equalRange_nsub_{nb_sub}',
+                '_model_type_loss', 'spmT_0002.nii'),
+            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}',
+                '_model_type_loss', '_threshold0', 'spmT_0001_thr.nii'),
+            join(f'l2_analysis_equalIndifference_nsub_{nb_sub}',
+                '_model_type_loss', 'spmT_0001.nii'),
+            join(f'l2_analysis_equalRange_nsub_{nb_sub}',
+                '_model_type_loss', '_threshold0', 'spmT_0001_thr.nii'),
+            join(f'l2_analysis_equalRange_nsub_{nb_sub}',
+                '_model_type_loss', 'spmT_0001.nii'),
+            join(f'l2_analysis_groupComp_nsub_{nb_sub}',
+                '_model_type_loss', '_threshold0', 'spmT_0001_thr.nii'),
+            join(f'l2_analysis_groupComp_nsub_{nb_sub}',
+                '_model_type_loss', 'spmT_0001.nii')
         ]
         return [join(self.directories.output_dir, f) for f in files]

narps_open/pipelines/team_R9K3_wip.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 # THIS IS A TEMPLATE THAT CAN BE USE TO REPRODUCE A NEW PIPELINE
 
 import os

narps_open/pipelines/team_T54A.py

@@ -765,17 +765,22 @@ def get_group_level_outputs(self):
             for parameter_values in parameter_sets]
 
         # Handle groupComp
-        files = [
-            'randomise_tfce_corrp_tstat1.nii.gz',
-            'randomise_tstat1.nii.gz',
-            'zstat1.nii.gz',
-            'tstat1.nii.gz'
+        parameters = {
+            'contrast_id': self.contrast_list,
+            'file': [
+                'randomise_tfce_corrp_tstat1.nii.gz',
+                'randomise_tstat1.nii.gz',
+                'zstat1.nii.gz',
+                'tstat1.nii.gz'
             ]
-
-        return_list += [join(
+        }
+        parameter_sets = product(*parameters.values())
+        template = join(
             self.directories.output_dir,
             f'group_level_analysis_groupComp_nsub_{len(self.subject_list)}',
-            '_contrast_id_2', f'{file}') for file in files]
+            '_contrast_id_{contrast_id}', '{file}')
+        return_list += [template.format(**dict(zip(parameters.keys(), parameter_values)))\
+            for parameter_values in parameter_sets]
 
         return return_list
 

narps_open/pipelines/team_V55J.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 from nipype.interfaces.spm import (Coregister, Smooth, OneSampleTTestDesign, EstimateModel, EstimateContrast, 
                                    Level1Design, TwoSampleTTestDesign, RealignUnwarp, 
                                    Normalize12, NewSegment, FieldMap)

narps_open/pipelines/team_X19V.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 from nipype.interfaces.fsl import (Info, ImageMaths, IsotropicSmooth, Threshold, Level1Design, FEATModel, 
                                    L2Model, Merge, FLAMEO, ContrastMgr, FILMGLS, MultipleRegressDesign, Cluster, BET, SmoothEstimate)
 from nipype.algorithms.modelgen import SpecifyModel

tests/conftest.py

@@ -10,6 +10,7 @@
 from os.path import join, isfile
 from shutil import rmtree
 
+from numpy import isclose
 from pytest import helpers
 from pathvalidate import is_valid_filepath
 
@@ -23,6 +24,15 @@
 # Init configuration, to ensure it is in testing mode
 Configuration(config_type='testing')
 
+@helpers.register
+def compare_float_2d_arrays(array_1, array_2):
+    """ Assert array_1 and array_2 are close enough """
+
+    assert len(array_1) == len(array_2)
+    for reference_array, test_array in zip(array_1, array_2):
+        assert len(reference_array) == len(test_array)
+        assert isclose(reference_array, test_array).all()
+
 @helpers.register
 def test_pipeline_outputs(pipeline: Pipeline, number_of_outputs: list):
     """ Test the outputs of a Pipeline.

tests/core/test_common.py

@@ -59,7 +59,7 @@ def test_remove_file(remove_test_dir):
 
         # Check file is removed
         assert not exists(test_file_path)
-    
+
     @staticmethod
     @mark.unit_test
     def test_remove_directory(remove_test_dir):
@@ -396,7 +396,7 @@ def test_node_list_to_file_1():
 
         # Check file was created
         assert exists(out_file)
- 
+
         # Check file was created
         with open(out_file, 'r', encoding = 'utf-8') as file:
             for list_element, file_element in zip(out_list, file.read().split('\n')):
@@ -429,9 +429,8 @@ def test_node_list_to_file_2():
 
         # Check file was created
         assert exists(out_file)
- 
+
         # Check file was created
         with open(out_file, 'r', encoding = 'utf-8') as file:
             for list_element, file_element in zip(out_list, file.read().split('\n')):
                 assert list_element == file_element
-

tests/data/test_results.py

@@ -12,8 +12,7 @@
 """
 
 from os.path import isdir, join
-from shutil import rmtree, move, copytree
-from time import sleep
+from shutil import rmtree, copytree
 
 from checksumdir import dirhash
 from pytest import mark
@@ -74,7 +73,8 @@ def fake_get_uid(_):
 
         # Mock the results path
         results_directory = Configuration()['directories']['narps_results']
-        Configuration()['directories']['narps_results'] = Configuration()['directories']['test_runs']
+        Configuration()[
+            'directories']['narps_results'] = Configuration()['directories']['test_runs']
 
         # Init & download the collection
         collection = ResultsCollection('2T6S')