diff --git a/README.md b/README.md index 2f874bb..1ad2f83 100644 --- a/README.md +++ b/README.md @@ -73,13 +73,10 @@ from ieegprep import RerefStruct Load, epoch and get the average of each stimulated electrode pair (minimizing memory usage) ``` -from ieegprep import load_data_epochs_averages +from ieegprep import load_data_epochs_averages, load_elec_stim_events -# seperate the different stimulated electrode-pairs (e.g. Ch01-Ch02...) and use each stim-pair as a trial "condition" -conditions_onsets = dict() -for row in events.itertuples(): - conditions_onsets.setdefault(row.electrical_stimulation_site, list()) - conditions_onsets[row.electrical_stimulation_site].append(float(row.onset)) +# load the events file and retrieve the different stimulated electrode-pairs (e.g. Ch01-Ch02...) as conditions +_, _, conditions_onsets, _ = load_elec_stim_events('/bids_data_root/subj-01/ieeg/sub-01_run-06_events.tsv') # retrieve epoch data averaged over conditions [srate, epochs, _] = load_data_epochs_averages('/bids_data_root/subj-01/ieeg/sub-01_run-06_ieeg.vhdr', diff --git a/ieegprep/__init__.py b/ieegprep/__init__.py index 40c7e6f..36626d7 100644 --- a/ieegprep/__init__.py +++ b/ieegprep/__init__.py @@ -47,5 +47,5 @@ def normalize_version(v): from ieegprep.bids.data_epoch import load_data_epochs, load_data_epochs_averages from ieegprep.bids.data_structure import list_bids_datasets from ieegprep.bids.rereferencing import RerefStruct -from ieegprep.bids.sidecars import load_event_info, load_stim_event_info, load_channel_info, load_ieeg_sidecar +from ieegprep.bids.sidecars import load_event_info, load_elec_stim_events, load_channel_info, load_ieeg_sidecar from ieegprep.fileio.IeegDataReader import VALID_FORMAT_EXTENSIONS, IeegDataReader diff --git a/ieegprep/bids/sidecars.py b/ieegprep/bids/sidecars.py index d4b0a15..58f5041 100644 --- a/ieegprep/bids/sidecars.py +++ b/ieegprep/bids/sidecars.py @@ -64,28 +64,39 @@ def load_event_info(filepath, addition_required_columns=None): raise FileNotFoundError('Could not find file') -def load_stim_event_info(filepath, additional_required_columns=None): +def load_elec_stim_events(filepath, additional_required_columns=None, exclude_bad_events=True, + concat_bidirectional_stimpairs=True, only_stimpairs_between_channels=None): """ Retrieve the electrical stimulation events from a _events.tsv file Args: - filepath (str): The path to the _events.tsv file to load - additional_required_columns(list/tuple): One or multiple additional columns that need to be present in the _events.tsv + filepath (str): The path to the _events.tsv file to load + additional_required_columns(list/tuple): One or multiple additional columns that need to be present in the _events.tsv + exclude_bad_events (bool): Whether to exclude events marked as 'bad' (only applied if a status column is present) + concat_bidirectional_stimpairs (bool): Whether to concatenate events that concern the same two stimulation + electrodes. If true, stimulation events between - for example - Ch01 and + Ch02 will be concatenated with stimulation events between Ch02 and Ch01. + only_stimpairs_between_channels(tuple/list): Only include stimulated pairs when both stimulated electrodes are included + in this (list/tuple) argument. Set to None to include all pairs. Returns: - trial_onsets (list) A list with the onsets of the stimulus events - trial_pairs (list) A list with the stim-pair names of each stimulus events - trials_bad_onsets If a status column exists in the events file, this list holds the onsets - of the trials that were marked as 'bad' and not included + trial_onsets (list) A list with the onsets of the stimulus events + trial_pairs (list) A list with the stim-pair names of each stimulus events + stimpair_onsets (dict) A dictionary that holds the onsets for each distinct stimulated electrode + pair. Each stimulated pair is an entry (e.g. Ch01-Ch02), and each entry + contains the corresponding onsets times as a list. + bad_trial_onsets If a status column exists in the events file, this list holds the onsets + of the trials that were marked as 'bad' and not included Raises: - RuntimeError: If the file could not be found, or if the mandatory 'onset', 'trial_type', - 'electrical_stimulation_site' column or any of the required additional - columns could not be found + RuntimeError: If the file could not be found, or if the mandatory 'onset', 'trial_type', + 'electrical_stimulation_site' column or any of the required additional + columns could not be found Note: This function expects the column 'trial_type' and 'electrical_stimulation_site' to exist in the _events.tsv file - according to the BIDS iEEG electrical stimulation specification. - Note 2: If a column status exists in the _events.tsv file, then these trials marked as 'bad' will be excluded + according to the BIDS iEEG electrical stimulation specification. Events of which the 'trial_type' are labelled + as 'electrical_stimulation' are regarded as electrical stimulation events. The 'electrical_stimulation_site' of + each stimulation event should indicate the stimulated electrodes separated by a dash, as such: Ch01-Ch02. """ @@ -105,7 +116,7 @@ def load_stim_event_info(filepath, additional_required_columns=None): # acquire the onset and electrode-pair for each stimulation trial_onsets = [] trial_pairs = [] - trials_bad_onsets = [] + bad_trial_onsets = [] trials_have_status = 'status' in events_tsv.columns for index, row in events_tsv.iterrows(): if row['trial_type'].lower() == 'electrical_stimulation': @@ -113,9 +124,9 @@ def load_stim_event_info(filepath, additional_required_columns=None): logging.warning('Invalid onset \'' + row['onset'] + '\' in events, should be a numeric value >= 0. Discarding trial...') continue - if trials_have_status: + if exclude_bad_events and trials_have_status: if not row['status'].lower() == 'good': - trials_bad_onsets.append(row['onset']) + bad_trial_onsets.append(row['onset']) continue pair = row['electrical_stimulation_site'].split('-') @@ -126,7 +137,51 @@ def load_stim_event_info(filepath, additional_required_columns=None): trial_onsets.append(float(row['onset'])) trial_pairs.append(pair) - return trial_onsets, trial_pairs, trials_bad_onsets + + # dictionary to hold the stimulated electrode pairs and their onsets + stimpairs_onsets = dict() + + # + if only_stimpairs_between_channels is None: + # include all stim-pairs + + for trial_index in range(len(trial_pairs)): + trial_pair = trial_pairs[trial_index] + if concat_bidirectional_stimpairs and (trial_pair[1] + '-' + trial_pair[0]) in stimpairs_onsets.keys(): + stimpairs_onsets[trial_pair[1] + '-' + trial_pair[0]].append(trial_onsets[trial_index]) + else: + if (trial_pair[0] + '-' + trial_pair[1]) in stimpairs_onsets.keys(): + stimpairs_onsets[trial_pair[0] + '-' + trial_pair[1]].append(trial_onsets[trial_index]) + else: + stimpairs_onsets[trial_pair[0] + '-' + trial_pair[1]] = [trial_onsets[trial_index]] + + else: + + # loop over all the combinations of included channels + # Note: only the combinations of stim-pairs that actually have events/trials end up in the output + for iChannel0 in range(len(only_stimpairs_between_channels)): + for iChannel1 in range(len(only_stimpairs_between_channels)): + + # retrieve the indices of all the trials that concern this stim-pair + indices = [] + if concat_bidirectional_stimpairs: + # allow concatenation of bidirectional pairs, pair order does not matter + if not iChannel1 < iChannel0: + # unique pairs while ignoring pair order + indices = [i for i, x in enumerate(trial_pairs) if + (x[0] == only_stimpairs_between_channels[iChannel0] and x[1] == only_stimpairs_between_channels[iChannel1]) or (x[0] == only_stimpairs_between_channels[iChannel1] and x[1] == only_stimpairs_between_channels[iChannel0])] + + else: + # do not concatenate bidirectional pairs, pair order matters + indices = [i for i, x in enumerate(trial_pairs) if + x[0] == only_stimpairs_between_channels[iChannel0] and x[1] == only_stimpairs_between_channels[iChannel1]] + + # add the pair if there are trials for it + if len(indices) > 0: + stimpairs_onsets[only_stimpairs_between_channels[iChannel0] + '-' + only_stimpairs_between_channels[iChannel1]] = [trial_onsets[i] for i in indices] + + # success, return the results + return trial_onsets, trial_pairs, stimpairs_onsets, bad_trial_onsets def load_ieeg_sidecar(filepath): @@ -193,6 +248,7 @@ def read_from_file(self, filepath, required_columns=None, interpret_numeric=True # This has the advantage of being able to allocate memory for the values for each of the columns input = open(filepath, mode="r", encoding="utf-8") rawText = input.read() + input.close() # split the lines, remove empty lines, and ensure there are columns in the first line lines = [line for line in rawText.split("\n") if line] diff --git a/tests/test_epoch_average_nopreproc_mem.py b/tests/test_epoch_average_nopreproc_mem.py index e894e9a..0f1fe80 100644 --- a/tests/test_epoch_average_nopreproc_mem.py +++ b/tests/test_epoch_average_nopreproc_mem.py @@ -13,7 +13,7 @@ import sys import unittest from ieegprep.bids.data_epoch import _prepare_input, _load_data_epoch_averages__by_channel_condition_trial, _load_data_epoch_averages__by_condition_trials, _load_data_epochs__by_channels__withPrep -from ieegprep.bids.sidecars import load_stim_event_info +from ieegprep.bids.sidecars import load_elec_stim_events from ieegprep.utils.console import ConsoleColors from ieegprep.utils.misc import clear_virtual_cache from memory_profiler import memory_usage, profile @@ -39,7 +39,7 @@ class TestEpochAverageNoPreProcMem(unittest.TestCase): # # by_channel_condition_trial # - ''' + def test01_epoch__by_channels__bv_multiplexed__no_preload_mem(self): test_name = 'Epoch & Average (no preprocessing), _load_data_epoch_averages__by_channel_condition_trial, Brainvision (multiplexed), no preload' self._run_test(test_name, self.bv_data_path, 'channel_condition_trial', preload_data=False, set_bv_orientation='MULTIPLEXED') @@ -108,7 +108,7 @@ def test15_epoch__by_condition_trials__mef__no_preload_mem(self): def test16_epoch__by_condition_trials__mef__preload_mem(self): test_name = 'Epoch & Average (no preprocessing), _load_data_epoch_averages__by_condition_trials, MEF, preloaded' self._run_test(test_name, self.mef_data_path, 'condition_trials', preload_data=True) - ''' + # # _load_data_epochs__by_channels__withPrep (mem) @@ -117,7 +117,7 @@ def test16_epoch__by_condition_trials__mef__preload_mem(self): def test17_epoch__by_prep_mem__bv_multiplexed__no_preload_mem(self): test_name = 'Epoch & Average (no preprocessing), _load_data_epochs__by_channels__withPrep (mem), Brainvision (multiplexed), no preload' self._run_test(test_name, self.bv_data_path, 'prep_mem', preload_data=False, set_bv_orientation='MULTIPLEXED') - ''' + def test18_epoch__by_prep_mem__bv_vectorized__no_preload_mem(self): test_name = 'Epoch & Average (no preprocessing), _load_data_epochs__by_channels__withPrep (mem), Brainvision (vectorized), no preload' self._run_test(test_name, self.bv_data_path, 'prep_mem', preload_data=False, set_bv_orientation='VECTORIZED') @@ -182,7 +182,6 @@ def test31_epoch__by_prep_mem__mef__no_preload_speed(self): def test32_epoch__by_prep_mem__mef__preload_speed(self): test_name = 'Epoch & Average (no preprocessing), _load_data_epochs__by_channels__withPrep (speed), MEF, preloaded' self._run_test(test_name, self.mef_data_path, 'prep_speed', preload_data=True) - ''' @profile @@ -212,11 +211,11 @@ def _prepare_and_epoch(self, data_path, by_routine, conditions_onsets, preload_d elif by_routine in ('prep_mem', 'prep_speed'): sampling_rate, data, _ = _load_data_epochs__by_channels__withPrep(True, data_reader, data_reader.channel_names, conditions_onsets, - trial_epoch=self.test_trial_epoch, - baseline_method=baseline_method, baseline_epoch=self.test_baseline_epoch, - out_of_bound_method=out_of_bound_method, metric_callbacks=None, - high_pass=False, early_reref=None, line_noise_removal=None, late_reref=None, - priority='mem' if by_routine == 'prep_mem' else 'speed') + trial_epoch=self.test_trial_epoch, + baseline_method=baseline_method, baseline_epoch=self.test_baseline_epoch, + out_of_bound_method=out_of_bound_method, metric_callbacks=None, + high_pass=False, early_reref=None, line_noise_removal=None, late_reref=None, + priority='mem' if by_routine == 'prep_mem' else 'speed') data_reader.close() @@ -229,23 +228,12 @@ def _run_test(self, test_name, data_path, by_routine, preload_data, set_bv_orien if set_bv_orientation is not None: print(' - set_bv_orientation: ' + set_bv_orientation) - # + # load the trial onsets for each of the stimulation conditions clear_virtual_cache() - trial_onsets, trial_pairs, _ = load_stim_event_info(data_path[0:data_path.rindex('_ieeg')] + '_events.tsv') - - # extract conditions - conditions_onsets = dict() - for trial_index in range(len(trial_pairs)): - trial_pair = trial_pairs[trial_index] - if (trial_pair[1] + '-' + trial_pair[0]) in conditions_onsets.keys(): - conditions_onsets[trial_pair[1] + '-' + trial_pair[0]].append(trial_onsets[trial_index]) - if (trial_pair[0] + '-' + trial_pair[1]) in conditions_onsets.keys(): - conditions_onsets[trial_pair[0] + '-' + trial_pair[1]].append(trial_onsets[trial_index]) - else: - conditions_onsets[trial_pair[0] + '-' + trial_pair[1]] = [trial_onsets[trial_index]] - conditions_onsets = list(conditions_onsets.values()) + _, _, conditions_onsets, _ = load_elec_stim_events(data_path[0:data_path.rindex('_ieeg')] + '_events.tsv', + concat_bidirectional_stimpairs=True) - # + # test the memory usage of the preparation and average epoch if set_bv_orientation is None: mem_usage = memory_usage((self._prepare_and_epoch, (data_path, by_routine, conditions_onsets, preload_data)), interval=0.005, include_children=True, multiprocess=True, max_usage=True) diff --git a/tests/test_epoch_average_nopreproc_perf.py b/tests/test_epoch_average_nopreproc_perf.py index c102db5..17aca26 100644 --- a/tests/test_epoch_average_nopreproc_perf.py +++ b/tests/test_epoch_average_nopreproc_perf.py @@ -15,7 +15,7 @@ from ieegprep.utils.console import ConsoleColors from ieegprep.utils.misc import time_func, clear_virtual_cache from ieegprep.bids.data_epoch import _prepare_input, _load_data_epoch_averages__by_channel_condition_trial, _load_data_epoch_averages__by_condition_trials, _load_data_epochs__by_channels__withPrep -from ieegprep.bids.sidecars import load_stim_event_info +from ieegprep.bids.sidecars import load_elec_stim_events class TestEpochAverageNoPreProcPerf(unittest.TestCase): @@ -247,20 +247,9 @@ def test_epoch_perf(self): tests[test_name]['cond_epoch_results_uncached'] = [] tests[test_name]['cond_epoch_results_cached'] = [] - # load the stim trial onsets - trial_onsets, trial_pairs, _ = load_stim_event_info(tests[test_name]['data_path'][0:tests[test_name]['data_path'].rindex("_ieeg")] + '_events.tsv') - - # extract conditions - conditions_onsets = dict() - for trial_index in range(len(trial_pairs)): - trial_pair = trial_pairs[trial_index] - if (trial_pair[1] + '-' + trial_pair[0]) in conditions_onsets.keys(): - conditions_onsets[trial_pair[1] + '-' + trial_pair[0]].append(trial_onsets[trial_index]) - if (trial_pair[0] + '-' + trial_pair[1]) in conditions_onsets.keys(): - conditions_onsets[trial_pair[0] + '-' + trial_pair[1]].append(trial_onsets[trial_index]) - else: - conditions_onsets[trial_pair[0] + '-' + trial_pair[1]] = [trial_onsets[trial_index]] - conditions_onsets = list(conditions_onsets.values()) + # load the trial onsets for each of the stimulation conditions + _, _, conditions_onsets, _ = load_elec_stim_events(tests[test_name]['data_path'][0:tests[test_name]['data_path'].rindex("_ieeg")] + '_events.tsv', + concat_bidirectional_stimpairs=True) # loop over the conditions for preload_condition in test['conditions']: diff --git a/tests/test_epoch_nopreproc_mem.py b/tests/test_epoch_nopreproc_mem.py index 7f89328..b435627 100644 --- a/tests/test_epoch_nopreproc_mem.py +++ b/tests/test_epoch_nopreproc_mem.py @@ -13,7 +13,7 @@ import sys import unittest from ieegprep.bids.data_epoch import _prepare_input, _load_data_epochs__by_channels, _load_data_epochs__by_trials, _load_data_epochs__by_channels__withPrep -from ieegprep.bids.sidecars import load_stim_event_info +from ieegprep.bids.sidecars import load_elec_stim_events from ieegprep.utils.console import ConsoleColors from ieegprep.utils.misc import clear_virtual_cache from memory_profiler import memory_usage, profile @@ -230,7 +230,7 @@ def _run_test(self, test_name, data_path, by_routine, preload_data, set_bv_orien # clear_virtual_cache() - trial_onsets, _, _ = load_stim_event_info(data_path[0:data_path.rindex('_ieeg')] + '_events.tsv') + trial_onsets, _, _, _ = load_elec_stim_events(data_path[0:data_path.rindex('_ieeg')] + '_events.tsv') # if set_bv_orientation is None: diff --git a/tests/test_epoch_nopreproc_perf.py b/tests/test_epoch_nopreproc_perf.py index 4f3e284..d31d318 100644 --- a/tests/test_epoch_nopreproc_perf.py +++ b/tests/test_epoch_nopreproc_perf.py @@ -15,7 +15,7 @@ from ieegprep.utils.console import ConsoleColors from ieegprep.utils.misc import time_func, clear_virtual_cache from ieegprep.bids.data_epoch import _prepare_input, _load_data_epochs__by_channels, _load_data_epochs__by_trials, _load_data_epochs__by_channels__withPrep -from ieegprep.bids.sidecars import load_stim_event_info +from ieegprep.bids.sidecars import load_elec_stim_events class TestEpochNoPreProcPerf(unittest.TestCase): @@ -247,7 +247,7 @@ def test_epoch_perf(self): tests[test_name]['cond_epoch_results_cached'] = [] # load the stim trial onsets - trial_onsets, _, _ = load_stim_event_info(tests[test_name]['data_path'][0:tests[test_name]['data_path'].rindex("_ieeg")] + '_events.tsv') + trial_onsets, _, _, _ = load_elec_stim_events(tests[test_name]['data_path'][0:tests[test_name]['data_path'].rindex("_ieeg")] + '_events.tsv') # loop over the conditions for preload_condition in test['conditions']: