4-classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification\n",
    "Notebook to use and test the machine learning models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-18T13:52:24.288831Z",
     "start_time": "2021-04-18T13:52:20.474527Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.model_selection import train_test_split, GroupKFold,cross_val_score, cross_validate,RandomizedSearchCV, GridSearchCV\n",
    "from sklearn.metrics import classification_report, confusion_matrix,make_scorer, balanced_accuracy_score,recall_score,roc_auc_score, roc_curve\n",
    "from sklearn.utils.class_weight import compute_sample_weight\n",
    "from sklearn import tree\n",
    "from sklearn.tree import DecisionTreeClassifier, plot_tree\n",
    "from xgboost import XGBClassifier\n",
    "import graphviz\n",
    "import random\n",
    "import pickle\n",
    "import os\n",
    "import gc\n",
    "from scipy.stats import entropy\n",
    "from sgkf_split import StratifiedGroupKFold     # Locally defined .py file.\n",
    "plt.style.use(\"seaborn\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-17T07:42:58.478738Z",
     "start_time": "2021-04-17T07:42:58.463738Z"
    }
   },
   "outputs": [],
   "source": [
    "def print_results(y_test, y_pred, label_names):\n",
    "    print(confusion_matrix(y_test, y_pred))\n",
    "    print(classification_report(y_test, y_pred,target_names=label_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-18T13:52:28.546938Z",
     "start_time": "2021-04-18T13:52:26.249752Z"
    }
   },
   "outputs": [],
   "source": [
    "y = np.load(os.path.join('Data','Final', 'y.npy')).astype(int).ravel()  # Convert to 1 and 0 rather than True or False.\n",
    "ids = np.load(os.path.join('Data','Final', 'ids.npy'))\n",
    "X = np.load(os.path.join('Data','Final', 'X.npy'))\n",
    "\n",
    "groups = ids[:,0]\n",
    "\n",
    "with open(os.path.join('Data','Final', 'col_names.txt')) as f:\n",
    "    col_names = f.read().splitlines()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision tree with K-fold CV\n",
    "Use the regular stratified K-fold approach."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T09:04:22.595629Z",
     "start_time": "2021-04-16T09:04:22.576669Z"
    }
   },
   "outputs": [],
   "source": [
    "mod_basic = DecisionTreeClassifier(max_depth=10, class_weight='balanced', random_state=1)\n",
    "\n",
    "scoring = {\n",
    "    'balanced_accuracy': 'balanced_accuracy',\n",
    "    'sensitivity': 'recall',\n",
    "    'specificity': make_scorer(recall_score,pos_label=0)\n",
    "}\n",
    "\n",
    "mod_basic_output = cross_validate(mod_basic, X, y, cv=5, scoring=scoring, return_estimator=True)\n",
    "print([mod_basic_output[key] for key in ['test_balanced_accuracy','test_sensitivity','test_specificity']])\n",
    "with open(os.path.join('Data','Final', 'mod_basic_1.pkl'), 'wb') as f:\n",
    "    pickle.dump(mod_basic_output, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T08:33:01.907169Z",
     "start_time": "2021-04-16T08:33:01.442019Z"
    }
   },
   "outputs": [],
   "source": [
    "plot_tree(model,max_depth=1)\n",
    "for i in [5,6,124,12,112,1]:\n",
    "    print(col_names[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T14:39:43.570449Z",
     "start_time": "2021-04-13T14:39:43.555899Z"
    }
   },
   "outputs": [],
   "source": [
    "tree.export_graphviz(model, out_file='tree_cut.dot', max_depth=2, feature_names=feature_cols, \\\n",
    "                             class_names=['No AF','AF'], filled=True)\n",
    "!dot -Tpng tree_cut.dot -o tree_cut.png   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T14:40:54.886761Z",
     "start_time": "2021-04-13T14:40:54.751297Z"
    }
   },
   "source": [
    "### Stratified Group K-Fold Cross Validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Decision Tree\n",
    "First use a decision tree to compare differences with the two K-fold methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-17T07:58:58.047583Z",
     "start_time": "2021-04-17T07:58:58.039716Z"
    }
   },
   "outputs": [],
   "source": [
    "from sgkf_split import StratifiedGroupKFold     # Locally defined .py file, taken from sklearn Github."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-17T08:04:08.626359Z",
     "start_time": "2021-04-17T07:59:18.098578Z"
    }
   },
   "outputs": [],
   "source": [
    "cv = StratifiedGroupKFold(n_splits=5)\n",
    "\n",
    "mod_grouped = DecisionTreeClassifier(max_depth=10, class_weight='balanced', random_state=15)\n",
    "\n",
    "scoring = {\n",
    "    'balanced_accuracy': 'balanced_accuracy',\n",
    "    'sensitivity': 'recall',\n",
    "    'specificity': make_scorer(recall_score,pos_label=0)\n",
    "}\n",
    "\n",
    "dt_grouped_output = cross_validate(mod_grouped, X, y, groups=groups, cv=cv, scoring=scoring, verbose=3, return_estimator=True)\n",
    "print([dt_grouped_output[key] for key in ['test_balanced_accuracy','test_sensitivity','test_specificity']])\n",
    "with open(os.path.join('Data','Final', 'dt_grouped_1.pkl'), 'wb') as f:\n",
    "    pickle.dump(dt_grouped_output, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exploration\n",
    "First run a number of models with different parameters to gauge risk of overfitting and the training times."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T10:42:36.627745Z",
     "start_time": "2021-04-16T10:42:36.609829Z"
    }
   },
   "outputs": [],
   "source": [
    "scale_pos_weight = (len(y)-np.sum(y))/np.sum(y)    # Account for imbalance by weighting positive class by (# Negative)/(# Positive)\n",
    "\n",
    "xgb = XGBClassifier(learning_rate=0.3, n_estimators=200, objective='binary:logistic', \\\n",
    "                    nthread=1, scale_pos_weight=scale_pos_weight, colsample_bytree=0.5, subsample=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T09:53:15.585084Z",
     "start_time": "2021-04-16T09:53:06.591551Z"
    }
   },
   "outputs": [],
   "source": [
    "train_indx, test_indx = next(cv.split(X, y, groups))\n",
    "train_sub_indx = random.sample(list(train_indx), int(len(train_indx)/4))\n",
    "X_train, X_test, y_train, y_test = X[train_sub_indx], X[test_indx], y[train_sub_indx], y[test_indx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T10:49:59.427618Z",
     "start_time": "2021-04-16T10:42:41.456789Z"
    }
   },
   "outputs": [],
   "source": [
    "eval_set = [(X_train, y_train), (X_test, y_test)]\n",
    "xgb.fit(X_train, y_train, eval_metric=\"auc\", eval_set=eval_set, verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-17T07:54:29.465649Z",
     "start_time": "2021-04-17T07:54:29.450747Z"
    }
   },
   "source": [
    "#### XGBoost with Group K-fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T13:03:32.398352Z",
     "start_time": "2021-04-16T11:16:50.489790Z"
    }
   },
   "outputs": [],
   "source": [
    "xgb = XGBClassifier(learning_rate=0.3, n_estimators=250, objective='binary:logistic', \\\n",
    "                    nthread=1, scale_pos_weight=scale_pos_weight, colsample_bytree=0.5, subsample=0.8, verbosity=3)\n",
    "\n",
    "scoring = {\n",
    "    'balanced_accuracy': 'balanced_accuracy',\n",
    "    'sensitivity': 'recall',\n",
    "    'specificity': make_scorer(recall_score,pos_label=0),\n",
    "    'roc_auc': 'roc_auc'\n",
    "}\n",
    "\n",
    "cv_output = cross_validate(xgb, X, y, groups=groups, cv=cv, scoring=scoring, verbose=10, return_estimator=True)\n",
    "cv_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T13:03:33.276712Z",
     "start_time": "2021-04-16T13:03:32.932518Z"
    }
   },
   "outputs": [],
   "source": [
    "# Save output and models to file.\n",
    "with open(os.path.join('Data','Final', 'XGB_results_1.pkl'), 'wb') as f:\n",
    "    pickle.dump(cv_output, f)\n",
    "# with open('XGB_results_1.pkl', 'rb') as f:\n",
    "#     cv_output_from_file = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T13:17:09.250667Z",
     "start_time": "2021-04-16T13:17:09.230083Z"
    }
   },
   "outputs": [],
   "source": [
    "print('test_balanced_accuracy: ', cv_output['test_balanced_accuracy'])\n",
    "print('test_sensitivity: \\t', cv_output['test_sensitivity'])\n",
    "print('test_specificity: \\t', cv_output['test_specificity'])\n",
    "print('test_roc_auc: \\t\\t', cv_output['test_roc_auc'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model Evaluation\n",
    "First get more details on one of the models - take the model with median balanced accuracy, which is the second model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T13:48:10.890604Z",
     "start_time": "2021-04-16T13:48:00.783608Z"
    }
   },
   "outputs": [],
   "source": [
    "cv_list = list(cv.split(X, y, groups))        # Get all Kfold splits.\n",
    "xgb_train_idx, xgb_test_idx = cv_list[1]      # Extract the split indices for the second model.\n",
    "\n",
    "X_test_xgb, y_test_xgb = X[xgb_test_idx], y[xgb_test_idx]\n",
    "\n",
    "y_pred_xgb = cv_output['estimator'][1].predict(X_test_xgb)    # Get predicted y labels for the test data.\n",
    "y_pred_xgb_p = cv_output['estimator'][1].predict_proba(X_test_xgb)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T13:48:11.851820Z",
     "start_time": "2021-04-16T13:48:11.180094Z"
    }
   },
   "outputs": [],
   "source": [
    "print_results(y_test_xgb, y_pred_xgb, ['No AF','AF'])    # Get the confusion matrix, precision and recall."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T13:51:05.832419Z",
     "start_time": "2021-04-16T13:51:05.735399Z"
    }
   },
   "source": [
    "#### ROC Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T13:56:54.498567Z",
     "start_time": "2021-04-16T13:56:54.360028Z"
    }
   },
   "outputs": [],
   "source": [
    "fpr, tpr, thresholds = roc_curve(y_test_xgb, y_pred_xgb_p)\n",
    "\n",
    "plt.plot(fpr, tpr)\n",
    "plt.xlim(0,0.02)\n",
    "\n",
    "plt.xlabel('FPR (1-Specificity)', fontsize=16)\n",
    "plt.ylabel('TPR (Sensitivity)', fontsize=16)\n",
    "plt.tick_params(axis='both', which='major', labelsize=12)\n",
    "plt.tick_params(axis='both', which='minor', labelsize=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature Importance\n",
    "Get average feature importance across the 5 models in the cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T13:35:09.888909Z",
     "start_time": "2021-04-16T13:35:06.258979Z"
    }
   },
   "outputs": [],
   "source": [
    "def mean_imp(imp):\n",
    "    df= pd.DataFrame([cv_output['estimator'][i].get_booster().get_score(importance_type=imp) for i in range(5)])\n",
    "    return df.mean()\n",
    "\n",
    "imp_type = ['weight','gain','cover','total_cover','total_gain']\n",
    "\n",
    "mean_importance_df = pd.DataFrame()\n",
    "\n",
    "for imp in imp_type:\n",
    "    mean_importance_df[imp] = mean_imp(imp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T14:19:40.393793Z",
     "start_time": "2021-04-16T14:19:40.370808Z"
    }
   },
   "outputs": [],
   "source": [
    "keys = ['f'+str(i) for i in range(len(col_names))]\n",
    "names_dict = dict(zip(keys, col_names))\n",
    "\n",
    "mean_importance_df['col_name'] = [names_dict[col] for col in mean_importance_df.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-16T14:20:35.294630Z",
     "start_time": "2021-04-16T14:20:34.977428Z"
    }
   },
   "outputs": [],
   "source": [
    "top_20 = mean_importance_df.sort_values(by='gain',ascending=False).head(20)\n",
    "\n",
    "fig,ax= plt.subplots(1,2, sharey=True, figsize=(8,8))\n",
    "\n",
    "ax[0].barh(top_20['col_name'], top_20['gain'], color='tab:blue', alpha=0.9)\n",
    "ax[0].set_xlabel('Gain', fontsize=16)\n",
    "ax[0].set_ylabel('Feature Name', fontsize=16)\n",
    "ax[0].tick_params(axis='both', which='major', labelsize=12)\n",
    "ax[0].tick_params(axis='both', which='minor', labelsize=10)\n",
    "\n",
    "ax[1].barh(top_20['col_name'], top_20['weight'], color='tab:brown', alpha=0.9)\n",
    "plt.gca().invert_yaxis()\n",
    "ax[1].set_xlabel('Weight', fontsize=16)\n",
    "ax[1].tick_params(axis='both', which='major', labelsize=12)\n",
    "ax[1].tick_params(axis='both', which='minor', labelsize=10)\n",
    "\n",
    "fig.tight_layout()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "439.664px",
    "left": "662px",
    "right": "20px",
    "top": "105px",
    "width": "655px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}