Update events load function name and added support for stim-pair output

README.md

@@ -73,13 +73,10 @@ from ieegprep import RerefStruct
 
 Load, epoch and get the average of each stimulated electrode pair (minimizing memory usage)
 ```
-from ieegprep import load_data_epochs_averages
+from ieegprep import load_data_epochs_averages, load_elec_stim_events
 
-# seperate the different stimulated electrode-pairs (e.g. Ch01-Ch02...) and use each stim-pair as a trial "condition"
-conditions_onsets = dict()
-for row in events.itertuples():
-    conditions_onsets.setdefault(row.electrical_stimulation_site, list())
-    conditions_onsets[row.electrical_stimulation_site].append(float(row.onset))
+# load the events file and retrieve the different stimulated electrode-pairs (e.g. Ch01-Ch02...) as conditions
+_, _, conditions_onsets, _ = load_elec_stim_events('/bids_data_root/subj-01/ieeg/sub-01_run-06_events.tsv')
 
 # retrieve epoch data averaged over conditions
 [srate, epochs, _] =  load_data_epochs_averages('/bids_data_root/subj-01/ieeg/sub-01_run-06_ieeg.vhdr',

ieegprep/__init__.py

@@ -47,5 +47,5 @@ def normalize_version(v):
 from ieegprep.bids.data_epoch import load_data_epochs, load_data_epochs_averages
 from ieegprep.bids.data_structure import list_bids_datasets
 from ieegprep.bids.rereferencing import RerefStruct
-from ieegprep.bids.sidecars import load_event_info, load_stim_event_info, load_channel_info, load_ieeg_sidecar
+from ieegprep.bids.sidecars import load_event_info, load_elec_stim_events, load_channel_info, load_ieeg_sidecar
 from ieegprep.fileio.IeegDataReader import VALID_FORMAT_EXTENSIONS, IeegDataReader

ieegprep/bids/sidecars.py

@@ -64,28 +64,39 @@ def load_event_info(filepath, addition_required_columns=None):
         raise FileNotFoundError('Could not find file')
 
 
-def load_stim_event_info(filepath, additional_required_columns=None):
+def load_elec_stim_events(filepath, additional_required_columns=None, exclude_bad_events=True,
+                          concat_bidirectional_stimpairs=True, only_stimpairs_between_channels=None):
     """
     Retrieve the electrical stimulation events from a _events.tsv file
 
     Args:
-        filepath (str):                           The path to the _events.tsv file to load
-        additional_required_columns(list/tuple):  One or multiple additional columns that need to be present in the _events.tsv
+        filepath (str):                              The path to the _events.tsv file to load
+        additional_required_columns(list/tuple):     One or multiple additional columns that need to be present in the _events.tsv
+        exclude_bad_events (bool):                   Whether to exclude events marked as 'bad' (only applied if a status column is present)
+        concat_bidirectional_stimpairs (bool):       Whether to concatenate events that concern the same two stimulation
+                                                     electrodes. If true, stimulation events between - for example - Ch01 and
+                                                     Ch02 will be concatenated with stimulation events between Ch02 and Ch01.
+        only_stimpairs_between_channels(tuple/list): Only include stimulated pairs when both stimulated electrodes are included
+                                                     in this (list/tuple) argument. Set to None to include all pairs.
 
     Returns:
-        trial_onsets (list)                       A list with the onsets of the stimulus events
-        trial_pairs (list)                        A list with the stim-pair names of each stimulus events
-        trials_bad_onsets                         If a status column exists in the events file, this list holds the onsets
-                                                  of the trials that were marked as 'bad' and not included
+        trial_onsets (list)                          A list with the onsets of the stimulus events
+        trial_pairs (list)                           A list with the stim-pair names of each stimulus events
+        stimpair_onsets (dict)                       A dictionary that holds the onsets for each distinct stimulated electrode
+                                                     pair. Each stimulated pair is an entry (e.g. Ch01-Ch02), and each entry
+                                                     contains the corresponding onsets times as a list.
+        bad_trial_onsets                             If a status column exists in the events file, this list holds the onsets
+                                                     of the trials that were marked as 'bad' and not included
 
     Raises:
-        RuntimeError:                             If the file could not be found, or if the mandatory 'onset', 'trial_type',
-                                                  'electrical_stimulation_site' column or any of the required additional
-                                                   columns could not be found
+        RuntimeError:                                If the file could not be found, or if the mandatory 'onset', 'trial_type',
+                                                     'electrical_stimulation_site' column or any of the required additional
+                                                     columns could not be found
 
     Note:   This function expects the column 'trial_type' and 'electrical_stimulation_site' to exist in the _events.tsv file
-            according to the BIDS iEEG electrical stimulation specification.
-    Note 2: If a column status exists in the _events.tsv file, then these trials marked as 'bad' will be excluded
+            according to the BIDS iEEG electrical stimulation specification. Events of which the 'trial_type' are labelled
+            as 'electrical_stimulation' are regarded as electrical stimulation events. The 'electrical_stimulation_site' of
+            each stimulation event should indicate the stimulated electrodes separated by a dash, as such: Ch01-Ch02.
 
     """
 
@@ -105,17 +116,17 @@ def load_stim_event_info(filepath, additional_required_columns=None):
     # acquire the onset and electrode-pair for each stimulation
     trial_onsets = []
     trial_pairs = []
-    trials_bad_onsets = []
+    bad_trial_onsets = []
     trials_have_status = 'status' in events_tsv.columns
     for index, row in events_tsv.iterrows():
         if row['trial_type'].lower() == 'electrical_stimulation':
             if not is_number(row['onset']) or isnan(float(row['onset'])) or float(row['onset']) < 0:
                 logging.warning('Invalid onset \'' + row['onset'] + '\' in events, should be a numeric value >= 0. Discarding trial...')
                 continue
 
-            if trials_have_status:
+            if exclude_bad_events and trials_have_status:
                 if not row['status'].lower() == 'good':
-                    trials_bad_onsets.append(row['onset'])
+                    bad_trial_onsets.append(row['onset'])
                     continue
 
             pair = row['electrical_stimulation_site'].split('-')
@@ -126,7 +137,51 @@ def load_stim_event_info(filepath, additional_required_columns=None):
             trial_onsets.append(float(row['onset']))
             trial_pairs.append(pair)
 
-    return trial_onsets, trial_pairs, trials_bad_onsets
+
+    # dictionary to hold the stimulated electrode pairs and their onsets
+    stimpairs_onsets = dict()
+
+    #
+    if only_stimpairs_between_channels is None:
+        # include all stim-pairs
+
+        for trial_index in range(len(trial_pairs)):
+            trial_pair = trial_pairs[trial_index]
+            if concat_bidirectional_stimpairs and (trial_pair[1] + '-' + trial_pair[0]) in stimpairs_onsets.keys():
+                stimpairs_onsets[trial_pair[1] + '-' + trial_pair[0]].append(trial_onsets[trial_index])
+            else:
+                if (trial_pair[0] + '-' + trial_pair[1]) in stimpairs_onsets.keys():
+                    stimpairs_onsets[trial_pair[0] + '-' + trial_pair[1]].append(trial_onsets[trial_index])
+                else:
+                    stimpairs_onsets[trial_pair[0] + '-' + trial_pair[1]] = [trial_onsets[trial_index]]
+
+    else:
+
+        # loop over all the combinations of included channels
+        # Note:     only the combinations of stim-pairs that actually have events/trials end up in the output
+        for iChannel0 in range(len(only_stimpairs_between_channels)):
+            for iChannel1 in range(len(only_stimpairs_between_channels)):
+
+                # retrieve the indices of all the trials that concern this stim-pair
+                indices = []
+                if concat_bidirectional_stimpairs:
+                    # allow concatenation of bidirectional pairs, pair order does not matter
+                    if not iChannel1 < iChannel0:
+                        # unique pairs while ignoring pair order
+                        indices = [i for i, x in enumerate(trial_pairs) if
+                                   (x[0] == only_stimpairs_between_channels[iChannel0] and x[1] == only_stimpairs_between_channels[iChannel1]) or (x[0] == only_stimpairs_between_channels[iChannel1] and x[1] == only_stimpairs_between_channels[iChannel0])]
+
+                else:
+                    # do not concatenate bidirectional pairs, pair order matters
+                    indices = [i for i, x in enumerate(trial_pairs) if
+                               x[0] == only_stimpairs_between_channels[iChannel0] and x[1] == only_stimpairs_between_channels[iChannel1]]
+
+                # add the pair if there are trials for it
+                if len(indices) > 0:
+                    stimpairs_onsets[only_stimpairs_between_channels[iChannel0] + '-' + only_stimpairs_between_channels[iChannel1]] = [trial_onsets[i] for i in indices]
+
+    # success, return the results
+    return trial_onsets, trial_pairs, stimpairs_onsets, bad_trial_onsets
 
 
 def load_ieeg_sidecar(filepath):
@@ -193,6 +248,7 @@ def read_from_file(self, filepath, required_columns=None, interpret_numeric=True
         # This has the advantage of being able to allocate memory for the values for each of the columns
         input = open(filepath, mode="r", encoding="utf-8")
         rawText = input.read()
+        input.close()
 
         # split the lines, remove empty lines, and ensure there are columns in the first line
         lines = [line for line in rawText.split("\n") if line]

tests/test_epoch_average_nopreproc_mem.py

@@ -13,7 +13,7 @@
 import sys
 import unittest
 from ieegprep.bids.data_epoch import _prepare_input, _load_data_epoch_averages__by_channel_condition_trial, _load_data_epoch_averages__by_condition_trials, _load_data_epochs__by_channels__withPrep
-from ieegprep.bids.sidecars import load_stim_event_info
+from ieegprep.bids.sidecars import load_elec_stim_events
 from ieegprep.utils.console import ConsoleColors
 from ieegprep.utils.misc import clear_virtual_cache
 from memory_profiler import memory_usage, profile
@@ -39,7 +39,7 @@ class TestEpochAverageNoPreProcMem(unittest.TestCase):
     #
     # by_channel_condition_trial
     #
-    '''
+
     def test01_epoch__by_channels__bv_multiplexed__no_preload_mem(self):
         test_name = 'Epoch & Average (no preprocessing), _load_data_epoch_averages__by_channel_condition_trial, Brainvision (multiplexed), no preload'
         self._run_test(test_name, self.bv_data_path, 'channel_condition_trial', preload_data=False, set_bv_orientation='MULTIPLEXED')
@@ -108,7 +108,7 @@ def test15_epoch__by_condition_trials__mef__no_preload_mem(self):
     def test16_epoch__by_condition_trials__mef__preload_mem(self):
         test_name = 'Epoch & Average (no preprocessing), _load_data_epoch_averages__by_condition_trials, MEF, preloaded'
         self._run_test(test_name, self.mef_data_path, 'condition_trials', preload_data=True)
-    '''
+
 
     #
     # _load_data_epochs__by_channels__withPrep (mem)
@@ -117,7 +117,7 @@ def test16_epoch__by_condition_trials__mef__preload_mem(self):
     def test17_epoch__by_prep_mem__bv_multiplexed__no_preload_mem(self):
         test_name = 'Epoch & Average (no preprocessing), _load_data_epochs__by_channels__withPrep (mem), Brainvision (multiplexed), no preload'
         self._run_test(test_name, self.bv_data_path, 'prep_mem', preload_data=False, set_bv_orientation='MULTIPLEXED')
-    '''
+
     def test18_epoch__by_prep_mem__bv_vectorized__no_preload_mem(self):
         test_name = 'Epoch & Average (no preprocessing), _load_data_epochs__by_channels__withPrep (mem), Brainvision (vectorized), no preload'
         self._run_test(test_name, self.bv_data_path, 'prep_mem', preload_data=False, set_bv_orientation='VECTORIZED')
@@ -182,7 +182,6 @@ def test31_epoch__by_prep_mem__mef__no_preload_speed(self):
     def test32_epoch__by_prep_mem__mef__preload_speed(self):
         test_name = 'Epoch & Average (no preprocessing), _load_data_epochs__by_channels__withPrep (speed), MEF, preloaded'
         self._run_test(test_name, self.mef_data_path, 'prep_speed', preload_data=True)
-    '''
 
 
     @profile
@@ -212,11 +211,11 @@ def _prepare_and_epoch(self, data_path, by_routine, conditions_onsets, preload_d
 
         elif by_routine in ('prep_mem', 'prep_speed'):
             sampling_rate, data, _ = _load_data_epochs__by_channels__withPrep(True, data_reader, data_reader.channel_names, conditions_onsets,
-                                                                           trial_epoch=self.test_trial_epoch,
-                                                                           baseline_method=baseline_method, baseline_epoch=self.test_baseline_epoch,
-                                                                           out_of_bound_method=out_of_bound_method, metric_callbacks=None,
-                                                                           high_pass=False, early_reref=None, line_noise_removal=None, late_reref=None,
-                                                                           priority='mem' if by_routine == 'prep_mem' else 'speed')
+                                                                              trial_epoch=self.test_trial_epoch,
+                                                                              baseline_method=baseline_method, baseline_epoch=self.test_baseline_epoch,
+                                                                              out_of_bound_method=out_of_bound_method, metric_callbacks=None,
+                                                                              high_pass=False, early_reref=None, line_noise_removal=None, late_reref=None,
+                                                                              priority='mem' if by_routine == 'prep_mem' else 'speed')
 
         data_reader.close()
 
@@ -229,23 +228,12 @@ def _run_test(self, test_name, data_path, by_routine, preload_data, set_bv_orien
         if set_bv_orientation is not None:
             print('  - set_bv_orientation: ' + set_bv_orientation)
 
-        #
+        # load the trial onsets for each of the stimulation conditions
         clear_virtual_cache()
-        trial_onsets, trial_pairs, _ = load_stim_event_info(data_path[0:data_path.rindex('_ieeg')] + '_events.tsv')
-
-        # extract conditions
-        conditions_onsets = dict()
-        for trial_index in range(len(trial_pairs)):
-            trial_pair = trial_pairs[trial_index]
-            if (trial_pair[1] + '-' + trial_pair[0]) in conditions_onsets.keys():
-                conditions_onsets[trial_pair[1] + '-' + trial_pair[0]].append(trial_onsets[trial_index])
-            if (trial_pair[0] + '-' + trial_pair[1]) in conditions_onsets.keys():
-                conditions_onsets[trial_pair[0] + '-' + trial_pair[1]].append(trial_onsets[trial_index])
-            else:
-                conditions_onsets[trial_pair[0] + '-' + trial_pair[1]] = [trial_onsets[trial_index]]
-        conditions_onsets = list(conditions_onsets.values())
+        _, _, conditions_onsets, _ = load_elec_stim_events(data_path[0:data_path.rindex('_ieeg')] + '_events.tsv',
+                                                                                concat_bidirectional_stimpairs=True)
 
-        #
+        # test the memory usage of the preparation and average epoch
         if set_bv_orientation is None:
             mem_usage = memory_usage((self._prepare_and_epoch, (data_path, by_routine, conditions_onsets, preload_data)),
                                      interval=0.005, include_children=True, multiprocess=True, max_usage=True)

tests/test_epoch_average_nopreproc_perf.py

@@ -15,7 +15,7 @@
 from ieegprep.utils.console import ConsoleColors
 from ieegprep.utils.misc import time_func, clear_virtual_cache
 from ieegprep.bids.data_epoch import _prepare_input, _load_data_epoch_averages__by_channel_condition_trial, _load_data_epoch_averages__by_condition_trials, _load_data_epochs__by_channels__withPrep
-from ieegprep.bids.sidecars import load_stim_event_info
+from ieegprep.bids.sidecars import load_elec_stim_events
 
 
 class TestEpochAverageNoPreProcPerf(unittest.TestCase):
@@ -247,20 +247,9 @@ def test_epoch_perf(self):
             tests[test_name]['cond_epoch_results_uncached'] = []
             tests[test_name]['cond_epoch_results_cached'] = []
 
-            # load the stim trial onsets
-            trial_onsets, trial_pairs, _ = load_stim_event_info(tests[test_name]['data_path'][0:tests[test_name]['data_path'].rindex("_ieeg")] + '_events.tsv')
-
-            # extract conditions
-            conditions_onsets = dict()
-            for trial_index in range(len(trial_pairs)):
-                trial_pair = trial_pairs[trial_index]
-                if (trial_pair[1] + '-' + trial_pair[0]) in conditions_onsets.keys():
-                    conditions_onsets[trial_pair[1] + '-' + trial_pair[0]].append(trial_onsets[trial_index])
-                if (trial_pair[0] + '-' + trial_pair[1]) in conditions_onsets.keys():
-                    conditions_onsets[trial_pair[0] + '-' + trial_pair[1]].append(trial_onsets[trial_index])
-                else:
-                    conditions_onsets[trial_pair[0] + '-' + trial_pair[1]] = [trial_onsets[trial_index]]
-            conditions_onsets = list(conditions_onsets.values())
+            # load the trial onsets for each of the stimulation conditions
+            _, _, conditions_onsets, _ = load_elec_stim_events(tests[test_name]['data_path'][0:tests[test_name]['data_path'].rindex("_ieeg")] + '_events.tsv',
+                                                                                    concat_bidirectional_stimpairs=True)
 
             # loop over the conditions
             for preload_condition in test['conditions']:

tests/test_epoch_nopreproc_mem.py

@@ -13,7 +13,7 @@
 import sys
 import unittest
 from ieegprep.bids.data_epoch import _prepare_input, _load_data_epochs__by_channels, _load_data_epochs__by_trials, _load_data_epochs__by_channels__withPrep
-from ieegprep.bids.sidecars import load_stim_event_info
+from ieegprep.bids.sidecars import load_elec_stim_events
 from ieegprep.utils.console import ConsoleColors
 from ieegprep.utils.misc import clear_virtual_cache
 from memory_profiler import memory_usage, profile
@@ -230,7 +230,7 @@ def _run_test(self, test_name, data_path, by_routine, preload_data, set_bv_orien
 
         #
         clear_virtual_cache()
-        trial_onsets, _, _ = load_stim_event_info(data_path[0:data_path.rindex('_ieeg')] + '_events.tsv')
+        trial_onsets, _, _, _ = load_elec_stim_events(data_path[0:data_path.rindex('_ieeg')] + '_events.tsv')
 
         #
         if set_bv_orientation is None:

tests/test_epoch_nopreproc_perf.py

@@ -15,7 +15,7 @@
 from ieegprep.utils.console import ConsoleColors
 from ieegprep.utils.misc import time_func, clear_virtual_cache
 from ieegprep.bids.data_epoch import _prepare_input, _load_data_epochs__by_channels, _load_data_epochs__by_trials, _load_data_epochs__by_channels__withPrep
-from ieegprep.bids.sidecars import load_stim_event_info
+from ieegprep.bids.sidecars import load_elec_stim_events
 
 
 class TestEpochNoPreProcPerf(unittest.TestCase):
@@ -247,7 +247,7 @@ def test_epoch_perf(self):
             tests[test_name]['cond_epoch_results_cached'] = []
 
             # load the stim trial onsets
-            trial_onsets, _, _ = load_stim_event_info(tests[test_name]['data_path'][0:tests[test_name]['data_path'].rindex("_ieeg")] + '_events.tsv')
+            trial_onsets, _, _, _ = load_elec_stim_events(tests[test_name]['data_path'][0:tests[test_name]['data_path'].rindex("_ieeg")] + '_events.tsv')
 
             # loop over the conditions
             for preload_condition in test['conditions']: