Implement loading of binary recordings

Python3/Binary.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+@author: malfatti
+@date: 2019-07-27
+
+Loads data recorded by Open Ephys in Binary format as numpy memmap.
+
+    DatLoad(Folder, Processor=None, Experiment=None, Recording=None, Unit='uV', ChannelMap=[])
+
+Parameters
+    Folder: str
+        Folder containing at least the subfolder 'experiment1'.
+
+    Processor: str or None, optional
+        Processor number to load, according to subsubsubfolders under
+        Folder>experimentX/recordingY/continuous . The number used is the one
+        after the processor name. For example, to load data from the folder
+        'Channel_Map-109_100.0' the value used should be '109'.
+        If not set, load all processors.
+
+    Experiment: int or None, optional
+        Experiment number to load, according to subfolders under Folder.
+        If not set, load all experiments.
+
+    Recording: int or None, optional
+        Recording number to load, according to subsubfolders under Folder>experimentX .
+        If not set, load all recordings.
+
+    Unit: str or None, optional
+        Unit to return the data, either 'uV' or 'mV' (case insensitive). In
+        both cases, return data in float32. Defaults to 'uV'.
+        If anything else, return data in int16.
+
+    ChannelMap: list, optional
+        If empty (default), load all channels.
+        If not empty, return only channels in ChannelMap, in the provided order.
+        CHANNELS ARE COUNTED STARTING AT 0.
+
+Returns:
+    Data: dict
+        Dictionary with data in the structure Data[Processor][Experiment][Recording].
+
+    Rate: dict
+        Dictionary with sampling rates in the structure Rate[Processor][Experiment].
+
+
+Example:
+    import Binary
+
+    Folder = '/home/user/PathToData/2019-07-27_00-00-00'
+    Data, Rate = Binary.Load(Folder)
+
+    ChannelMap = [0,15,1,14]
+    Recording = 3
+    Data2, Rate2 = Binary.Load(Folder, Recording=Recording, ChannelMap=ChannelMap, Unit='Bits')
+
+
+Warning:
+    Data placed inside dictionaries is affected when passed to functions.
+    Therefore, if you do, for example:
+
+        FilteredData = MyFilterFunction(Data['100']['0']['0'], Rate['100']['0'], Frequencies)
+
+    the data in Data['100']['0']['0'] will be filtered! To avoid this behaviour, pass the
+    data with the .copy() method:
+
+        FilteredData = MyFilterFunction(Data['100']['0']['0'].copy(), Rate['100']['0'], Frequencies)
+
+    then the data in Data['100']['0']['0'] will remain unaltered.
+
+"""
+#%%
+import numpy as np
+from ast import literal_eval
+from glob import glob
+
+
+def ApplyChannelMap(Data, ChannelMap):
+    print('Retrieving channels according to ChannelMap... ', end='')
+    for R, Rec in Data.items():
+        if Rec.shape[1] < len(ChannelMap) or max(ChannelMap) > Rec.shape[1]-1:
+            print('')
+            print('Not enough channels in data to apply channel map. Skipping...')
+            continue
+
+        Data[R] = Data[R][:,ChannelMap]
+
+    return(Data)
+
+
+def BitsToVolts(Data, ChInfo, Unit):
+    print('Converting to uV... ', end='')
+    Data = {R: Rec.astype('float32') for R, Rec in Data.items()}
+
+    if Unit.lower() == 'uv': U = 1
+    elif Unit.lower() == 'mv': U = 10**-3
+
+    for R in Data.keys():
+        for C in range(len(ChInfo)):
+            Data[R][:,C] = Data[R][:,C] * ChInfo[C]['bit_volts'] * U
+            if 'ADC' in ChInfo[C]['channel_name']: Data[R][:,C] *= 10**6
+
+    return(Data)
+
+
+def Load(Folder, Processor=None, Experiment=None, Recording=None, Unit='uV', ChannelMap=[]):
+    Files = sorted(glob(Folder+'/**/*.dat', recursive=True))
+    InfoFiles = sorted(glob(Folder+'/*/*/structure.oebin'))
+
+
+    Data, Rate = {}, {}
+    for F,File in enumerate(Files):
+        Exp, Rec, _, Proc = File.split('/')[-5:-1]
+        Exp = str(int(Exp[10:])-1)
+        Rec = str(int(Rec[9:])-1)
+        Proc = Proc.split('.')[0].split('-')[-1]
+        if '_' in Proc: Proc = Proc.split('_')[0]
+
+        if Proc not in Data.keys(): Data[Proc], Rate[Proc] = {}, {}
+
+        if Experiment:
+            if int(Exp) != Experiment-1: continue
+
+        if Recording:
+            if int(Rec) != Recording-1: continue
+
+        if Processor:
+            if Proc != Processor: continue
+
+        print('Loading recording', int(Rec)+1, '...')
+        if Exp not in Data[Proc]: Data[Proc][Exp] = {}
+        Data[Proc][Exp][Rec] = np.memmap(File, dtype='int16')
+
+
+        Info = literal_eval(open(InfoFiles[F]).read())
+        ProcIndex = [Info['continuous'].index(_) for _ in Info['continuous']
+                     if str(_['recorded_processor_id']) == Proc][0]
+
+        ChNo = Info['continuous'][ProcIndex]['num_channels']
+        if Data[Proc][Exp][Rec].shape[0]%ChNo:
+            print('Rec', Rec, 'is broken')
+            del(Data[Proc][Exp][Rec])
+            continue
+
+        SamplesPerCh = Data[Proc][Exp][Rec].shape[0]//ChNo
+        Data[Proc][Exp][Rec] = Data[Proc][Exp][Rec].reshape((SamplesPerCh, ChNo))
+        Rate[Proc][Exp] = Info['continuous'][ProcIndex]['sample_rate']
+
+    for Proc in Data.keys():
+        for Exp in Data[Proc].keys():
+            if Unit.lower() in ['uv', 'mv']:
+                ChInfo = Info['continuous'][ProcIndex]['channels']
+                Data[Proc][Exp] = BitsToVolts(Data[Proc][Exp], ChInfo, Unit)
+
+            if ChannelMap: Data[Proc][Exp] = ApplyChannelMap(Data[Proc][Exp], ChannelMap)
+
+    print('Done.')
+
+    return(Data, Rate)
+