1-processing-raw-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combining Data\n",
    "Takes all of the individual 804 recording files and combines them into one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T09:55:02.150099Z",
     "start_time": "2021-02-07T09:55:00.858879Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "\n",
    "# Get directory paths.\n",
    "ecg_path = os.path.join('Data', 'ECG_Data')\n",
    "class_path = os.path.join('Data', 'Class')\n",
    "\n",
    "# Get lists of files.\n",
    "ecg_files = [f for f in os.listdir(ecg_path) if os.path.isfile(\n",
    "            os.path.join(ecg_path, f))]\n",
    "control_files = [f for f in os.listdir(class_path) if os.path.isfile(\n",
    "            os.path.join(class_path, f))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T09:57:26.757976Z",
     "start_time": "2021-02-07T09:57:26.734267Z"
    }
   },
   "outputs": [],
   "source": [
    "file_nums = [int(re.findall(\"\\d+\", f)[0]) for f in ecg_files]\n",
    "\n",
    "print(len(file_nums))\n",
    "print(len(np.unique(file_nums)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T10:02:30.995573Z",
     "start_time": "2021-02-07T10:02:13.953273Z"
    }
   },
   "outputs": [],
   "source": [
    "cols = ['time', 'interval', 'beat_type', 'annotation']\n",
    "df_list = []\n",
    "\n",
    "for filename in ecg_files[3:6]:\n",
    "    print(filename)\n",
    "    file_num = int(re.findall(\"\\d+\", filename)[0])         # Add a file number column extracted from the filename.\n",
    "\n",
    "    # Read in each file, with each line read in as one whole string.\n",
    "    tmp = pd.read_table(os.path.join('Data','ECG_Data',filename))\n",
    "    \n",
    "    # First split the line into pieces using regex and capturing groups (time, R-R interval, beat type & annotation)\n",
    "    # From this, take the first two fields, and strip whitespace from the annotation, and add the file & row numbers.\n",
    "    new_lines = [(file_num, i, tup[0], int(tup[1]), tup[3].strip()) for i, tup in enumerate ( \\\n",
    "                      [re.findall('([\\d:]+) (\\d+) (\\w+) (.+)', tmp.iloc[i][0])[0] for i in range(len(tmp))])]\n",
    "    \n",
    "    adj_new_lines = [(t[0],t[1]) for t in new_lines]\n",
    "    \n",
    "    df_list.extend(adj_new_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T10:03:14.454355Z",
     "start_time": "2021-02-07T10:03:14.411354Z"
    }
   },
   "outputs": [],
   "source": [
    "[t for t in df_list if t[1] < 30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['time', 'interval', 'beat_type', 'annotation']\n",
    "df_list = []\n",
    "\n",
    "for filename in ecg_files:\n",
    "    print(filename)\n",
    "    file_num = int(re.findall(\"\\d+\", filename)[0])         # Add a file number column extracted from the filename.\n",
    "\n",
    "    # Read in each file, with each line read in as one whole string.\n",
    "    tmp = pd.read_table(os.path.join('Data','ECG_Data',filename))\n",
    "    \n",
    "    # First split the line into pieces using regex and capturing groups (time, R-R interval, beat type & annotation)\n",
    "    # From this, take the first two fields, and strip whitespace from the annotation, and add the file & row numbers.\n",
    "    new_lines = [(file_num, i, tup[0], int(tup[1]), tup[3].strip()) for i, tup in enumerate ( \\\n",
    "                      [re.findall('([\\d:]+) (\\d+) (\\w+) (.+)', tmp.iloc[i][0])[0] for i in range(len(tmp))])]\n",
    "    \n",
    "    df_list.extend(new_lines)\n",
    "    \n",
    "df = pd.DataFrame(df_list, columns = ['file_num','row_num','time', 'interval', 'annotation'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.to_csv(dirpath+'combined_series.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T12:31:06.007584Z",
     "start_time": "2021-01-27T12:31:05.997759Z"
    }
   },
   "outputs": [],
   "source": [
    "cols_controls = ['time','control']\n",
    "# df_controls = pd.read_csv(dirpath +'combined_controls.txt', delim_whitespace=True, names=cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T12:39:48.238633Z",
     "start_time": "2021-01-27T12:39:33.304777Z"
    }
   },
   "outputs": [],
   "source": [
    "outfilepath = os.path.join('Data', 'combined_controls.csv')\n",
    "\n",
    "for file in control_files:\n",
    "    df = pd.read_csv(os.path.join(class_path, file), \n",
    "                    delim_whitespace=True, names=cols_controls, \n",
    "                    dtype={'time':'object','control':'int8'})\n",
    "    df['file_num'] = int(re.findall(\"\\d+\", file)[0]) \n",
    "        \n",
    "#     df.to_csv(outfilepath, mode='a', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T12:48:05.716129Z",
     "start_time": "2021-01-27T12:47:04.205743Z"
    }
   },
   "outputs": [],
   "source": [
    "ecg_df = pd.read_csv(os.path.join('Data', 'combined_series.csv'), dtype={\n",
    "                     'file_num': 'int16', 'row_num': 'uint32', 'time': 'object', 'interval': 'int16', 'annotation':'object'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T12:48:39.630199Z",
     "start_time": "2021-01-27T12:48:38.357922Z"
    }
   },
   "outputs": [],
   "source": [
    "control_df = pd.read_csv(os.path.join('Data', 'combined_controls.csv'), dtype={\n",
    "                         'file_num': 'int16', 'time': 'object', 'control': 'int8'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2021-01-27T12:49:08.045Z"
    }
   },
   "outputs": [],
   "source": [
    "ecg_df.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "364.667px",
    "left": "292px",
    "right": "20px",
    "top": "90px",
    "width": "596px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}