SVM_Classification (2).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#In this notebook we are going to use SVM to predict which repair tools could have generated a specific commit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# necesseray imports\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "import seaborn as sn\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt  \n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of the data sate is:  (75, 70)\n",
      "We have 70 features for each of the 75 DRR data points\n",
      "Below wee can see how many data points of each tool we had\n",
      "jaid          18\n",
      "arja          12\n",
      "SimFix        11\n",
      "ACS           10\n",
      "nopol          4\n",
      "elixir         4\n",
      "DeepRepair     4\n",
      "SequenceR      4\n",
      "ssFix          3\n",
      "SketchFix      2\n",
      "CapGen         2\n",
      "genprog        1\n",
      "Name: tool, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Read the data\n",
    "# This is DRR features manually selected to reduce the computation complexityn.\n",
    "# This a link to the features: https://github.com/SpoonLabs/coming-experiments/tree/master/out_features_d4j\n",
    "\n",
    "data=pd.read_csv(\"/Users/macbook/PycharmProjects/ML/megadiff-correct.csv\")\n",
    "print(\"shape of the data sate is: \",data.shape)\n",
    "print(\"We have 70 features for each of the 75 DRR data points\")\n",
    "print(\"Below wee can see how many data points of each tool we had\")\n",
    "print(data['tool'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "Dir=\"/Users/macbook/Documents/university/internship/MachineLearningOnComingData/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "tools_list=data['tool'].unique()\n",
    "dummy=pd.get_dummies(data['tool'])\n",
    "\n",
    "X = data.drop('tool', axis=1,inplace=True)\n",
    "X = data.drop('file_name', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ACS' 'arja' 'CapGen' 'DeepRepair' 'elixir' 'genprog' 'jaid' 'nopol'\n",
      " 'SequenceR' 'SimFix' 'SketchFix' 'ssFix']\n"
     ]
    }
   ],
   "source": [
    "print(tools_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tools are studied seperately to make it easier to review the graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: 0.634146\n",
      "ROC AUC: 0.928571\n",
      "Accuracy: 0.933333\n",
      "Precision: 0.500000\n",
      "Recall: 1.000000\n",
      "F1 score: 0.666667\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEWCAYAAAB7QRxFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcHFW5xvHfE0CEEJaENQSIrC4gyBLWiyAaAdlEhIRdkAgqF67KBYULguJyUZRVCLLIEiSKAYQIRBQh3ABZCBAWWWVJwhoIq5DMvPePOgOdTs9Md0/XdNfk+eZTn+k+VV3n9EznnTNvnTpHEYGZmRVXv2Y3wMzMesaB3Mys4BzIzcwKzoHczKzgHMjNzArOgdzMrOAcyK3HJC0l6c+S5kr6Qw/Oc4CkWxvZtmaQ9BdJhzS7HbbocCBfhEjaX9IUSW9Jmp0CznYNOPU+wCrAoIj4ar0niYirImJ4A9qzAEk7SApJ48rKN07lt1d5nh9KurK74yJil4j4XZ3NNauZA/kiQtJ3gF8DPyELumsC5wN7NuD0awGPRcT8BpwrLy8DW0saVFJ2CPBYoypQxv+nrNf5Q7cIkLQccBrwrYj4U0S8HRHzIuLPEXFcOmZJSb+WNCttv5a0ZNq3g6TnJX1X0kupN/+1tO9U4GRgv9TTP7y85yppaOr5Lp6eHyrpKUlvSnpa0gEl5RNLXreNpMkpZTNZ0jYl+26X9CNJd6Xz3CppxS6+De8D1wEj0usXA/YDrir7Xp0l6TlJb0iaKuk/UvnOwA9K3uf9Je04XdJdwDvA2qns62n/byRdW3L+n0u6TZKq/gGadcOBfNGwNfBRYFwXx5wIbAVsAmwMDANOKtm/KrAcsDpwOHCepBUi4hSyXv41EbFMRFzcVUMk9QfOBnaJiAHANsD0CscNBG5Kxw4CzgRuKutR7w98DVgZ+Ajwva7qBi4HDk6PvwjMAGaVHTOZ7HswEBgD/EHSRyPi5rL3uXHJaw4CRgEDgGfKzvddYKP0S+o/yL53h4TnxrAGciBfNAwCXukm9XEAcFpEvBQRLwOnkgWoDvPS/nkRMR54C9igzva0AxtKWioiZkfEQxWO+RLweERcERHzI+Jq4FFg95JjLo2IxyLiXWAsWQDuVET8HzBQ0gZkAf3yCsdcGRGvpjp/CSxJ9+/zsoh4KL1mXtn53iH7Pp4JXAkcHRHPd3M+s5o4kC8aXgVW7EhtdGIwC/Ymn0llH5yj7BfBO8AytTYkIt4mS2kcCcyWdJOkj1fRno42rV7y/IU62nMF8G1gRyr8hSLpe5IeSemc18n+CukqZQPwXFc7I+Ie4ClAZL9wzBrKgXzRMAl4D9iri2NmkV207LAmC6cdqvU2sHTJ81VLd0bELRHxBWA1sl72RVW0p6NNM+tsU4crgG8C41Nv+QMp9fHfwL7AChGxPDCXLAADdJYO6TJNIulbZD37Wen8Zg3lQL4IiIi5ZBckz5O0l6SlJS0haRdJ/5sOuxo4SdJK6aLhyWSpgHpMB7aXtGa60Pr9jh2SVpG0Z8qVv0eWommvcI7xwPppyOTikvYDPgncWGebAIiIp4HPkl0TKDcAmE82wmVxSScDy5bsfxEYWsvIFEnrAz8GDiRLsfy3pC5TQGa1ciBfRKR873fILmC+TJYO+DbZSA7Igs0U4AHgQWBaKqunrgnANelcU1kw+PZL7ZgFzCELqkdVOMerwG5kFwtfJevJ7hYRr9TTprJzT4yISn9t3ALcTDYk8Rng3yyYNum42elVSdO6qyelsq4Efh4R90fE42QjX67oGBFk1gjyxXMzs2Jzj9zMrOAcyM3MCs6B3Mys4BzIzcwKrqsbRJpq3itP+SqsLWSd9Rsxx5f1Nc/OebDHc9fUEnOWWHHtlporxz1yM7OCa9keuZlZr2pva3YL6uZAbmYG0NbK0+l3zYHczAyIqDRTRDE4kJuZAbQXN5D7YqeZGUC0V791Q9IlaTWtGSVlZ0h6VNIDksZJWr6T1/5L0oOSpkuaUk3THcjNzCC72Fnt1r3LgJ3LyiYAG0bEp8kmZvt++YtK7BgRm0TE5tVU5kBuZgYN7ZFHxB1ks3uWlt1asjjL3cCQRjXdgdzMDIi2+VVvkkZJmlKyjaqxusOAv3TWFODWtPh3Vef1xU4zM6jpYmdEjAZG11ONpBPJFjC5qpNDtouImZJWBiZIejT18DvlHrmZGTQ0tdIZSYeSLZhyQHSyGEREzExfXyJbV3ZYd+d1IDczg0Zf7FyIpJ3JVrrao3y92JJj+ksa0PEYGA7MqHRsKQdyMzNo9PDDq8kWPd9A0vOSDgfOJVsXdkIaWnhBOnawpPHppasAEyXdD9wL3BQRN3dXn3PkZmbQ0Fv0I2JkheKLOzl2FrBrevwUsHGt9TmQm5lBoe/sdCA3MwMiPPuhmVmxedIsM7OCc2rFzKzg3CM3Myu4tnnNbkHdHMjNzMCpFTOzwnNqxcys4NwjNzMrOAdyM7NiC1/sNDMrOOfIzcwKzqkVM7OCc4/czKzg3CM3Mys498jNzApufuMWluhtDuRmZuAeuZlZ4TlHbmZWcO6Rm5kVnHvkZmYF5x65mVnBedSKmVnBRTS7BXVzIDczA+fIzcwKz4HczKzgfLHTzKzg2tqa3YK6OZCbmYFTK2ZmhedAbmZWcM6Rm5kVW7QXdxx5v2Y3wMysJbS3V791Q9Ilkl6SNKOkbKCkCZIeT19X6OS1h6RjHpd0SDVNdyA3M4Ns1Eq1W/cuA3YuKzsBuC0i1gNuS88XIGkgcAqwJTAMOKWzgF/KgdzMDBraI4+IO4A5ZcV7Ar9Lj38H7FXhpV8EJkTEnIh4DZjAwr8QFuIcuZkZ9MaolVUiYnZ6/AKwSoVjVgeeK3n+fCrrkgN5iznpJ2dyx133MnCF5bnuygsAOGf05fxt4iT6qR8DV1iO00/8LiuvNKjJLbVmOeOc09hp+Pa8+socvrDt3s1uTt9Rw6RZkkYBo0qKRkfE6OqripDUsKurTq20mL12/QIXnPnjBcq+dsBXGHf5b7j2d+fx2W235DeXjmlS66wV/GHM9Rz81aOa3Yy+p4bUSkSMjojNS7ZqgviLklYDSF9fqnDMTGCNkudDUlmXHMhbzOabbMRyyw5YoGyZ/v0/ePzuu/9G6u1WWSu5d9JUXn9tbrOb0fe0R/VbfW4AOkahHAJcX+GYW4DhklZIFzmHp7Iu5ZZakbQScAQwtLSeiDgsrzr7srMuvIwbbr6NAf37c8k5P2t2c8z6ngbOtSLpamAHYEVJz5ONRPkZMFbS4cAzwL7p2M2BIyPi6xExR9KPgMnpVKdFRPlF04Xk2SO/HlgO+CtwU8nWKUmjJE2RNOW3l1+dY9OK55hvHMpt467gS8N3ZMy1f252c8z6nGhvr3rr9lwRIyNitYhYIiKGRMTFEfFqROwUEetFxOc7AnRETImIr5e89pKIWDdtl1bT9jwvdi4dEcfX8oKUZxoNMO+Vp4p7m1WOdhu+I0d972S+/fWDmt0Us77Fd3ZWdKOkXXM8/yLjmec+vNbxtzsn8bG1hjSxNWZ9VLRXv7WYPHvkxwA/kPQ+8D4gslE3y+ZYZ+Edd8rPmHzfA7z++hvstNeBfPPwg7hz0mT+9ezzqJ8YvOrKnHzc0c1upjXRORf9nK233YIVBi3PPTP+ypk/O49rrhzX7GYVX4F75IoWXXDUqRWrZJ3192x2E6wFPTvnwR6P5Xr75BFVx5z+p/2+pcaO5ZZaUeZASf+Tnq8haVhe9ZmZ9UiBUyt55sjPB7YG9k/P3wLOy7E+M7P65T+OPDd55si3jIhNJd0HEBGvSfpIjvWZmdWtmmGFrSrPQD5P0mJAwAc3CBX3O2VmfVsL9rSrlWcgPxsYB6ws6XRgH+CkHOszM6ufA/nCIuIqSVOBnciGHu4VEY/kVZ+ZWY808Bb93pbnXCsDyWb3urqkbImImJdXnWZm9Srymp15plamkU3H+BpZj3x54AVJLwJHRMTUHOs2M6tNgQN5nsMPJwC7RsSKETEI2AW4Efgm2dBEM7PW0cCl3npbnoF8q4j4YB7diLgV2Doi7gaWzLFeM7PaeRx5RbMlHQ/8Pj3fj2yFjMXwMEQzazUtGKCrlWcg359sMvXr0vOJqWwx0oTqZmatItqK27/Mc/jhK8DRkK1PV7J6NMATedVrZlaXAvfIe2vNzi5XBjIza7Zoj6q3VpNnaqVUS035aGa2kBYM0NXqrUB+US/VY2ZWn+KmyBufWpG0tKQlSp5vACwpae9G12Vm1igxv73qrdXkkSO/GRgKIGldYBKwNvAtST/NoT4zs55rr2FrMXmkVlaIiMfT40OAqyPi6DQX+VTg+znUaWbWI614EbNaefTIS78bnyO7VZ+IeJ+W/F1mZoZ75GUekPQLYCawLnArgKTlc6jLzKwh3CNf0BHAK2R58uER8U4q/yTwixzqMzPrOffIF7AM8OeIeKisfC7ZhVAzs5YT85vdgvrl0SM/BxhUoXwgcFYO9ZmZ9Vi0V7+1mm4DuaS9JQ1Ij0+QNFbSJl28ZN2IuKO8MCLuBD5df1PNzHJU4NRKNT3yH0bEm5K2AXYFrgIu6OL4AV3sW6KLfWZmTdOne+RAx4qkuwEXRsT1dL0wxBOSdi0vlLQL8FTtTTQzy1+RA3k1FztnSzoP2BnYPN3Y09UvgGOBmyTtS3YDEMDmwNZkvwzMzFpOtBV3br9qeuT7Av8AvhQRrwErAid0dnC6q3Oj9JqhwFrA7cDXgWN61lwzs3wUuUfeaSCXtKykZdMxNwOz0vO3gLu6OmlEvBcRl5Ll04NspaBTgUca1XAzs0aKdlW9dUXSBpKml2xvSDq27JgdJM0tOebknrS9q9TKQ2RBuLTVHc8DWLOTN7E+MDJtrwDXAIqIHXvSUDOzPDWqpx0R/wQ2AUhrFM8ExlU49M6IaEi6udNAHhFr1HnOR4E7gd0i4gkASf9V57nMzHpFRC458p2AJyPimTxO3qGqG4IkjZD0g/R4iKTNujh8b2A28HdJF0naCa8QZGYtrpYcuaRRkqaUbKM6Oe0I4OpO9m0t6X5Jf5H0qZ60vdtRK5LOJRv/vT3wE+AdsnHkW1Q6PiKuA66T1B/Yk2wUy8qSfgOMi4hbe9JgM7M8tNcwaiUiRgOjuzomjfDbg8pTd08D1oqIt9Jw7euA9apv7YKq6ZFvExHfAP4NEBFzgI9096KIeDsixkTE7sAQ4D7g+HobamaWp0Zd7CyxCzAtIl5cqK6INyLirfR4PLCEpBXrbXs1gXyepH6kecYlDaLGm1Qj4rWIGB0RO9XRRjOz3OUQyEfSSVpF0qqSlB4PI4vFr9bb9mpuCDoPuBZYSdKpZOPKT623QjOzVhQNnI48pZa/AHyjpOzIrJ64ANgHOErSfOBdYERE/S3oNpBHxOWSpgKfT0VfjYgZ9VZoZtaKauhpd3+uiLcpmwU2BfCOx+cC5zaqvmrnI18MmEeWXslj6lszs6bKafhhr6hmGtsTyfI8g8kuWo6R5AWUzaxPaWtT1VurqaZHfjDwmY4l2ySdTjYC5ad5NszMrDcVuUde1eyHZcctnsrMzPqMRubIe1ungVzSr8hy4nOAhyTdkp4PByb3TvPMzHpHI0et9LaueuQdI1MeAm4qKb87v+aYmTVHn+yRR8TFvdkQM7Nmamsv7oC8auZaWQc4Hfgk8NGO8ohYP8d2mZn1qiKnVqr5FXQZcCnZDIa7AGPJ5hg3M+sz2kNVb62mmkC+dETcAhART0bESWQB3cysz4hQ1VurqWb44Xtp0qwn01wBM4EB+TbLzKx3FTm1Uk0g/y+gP/CfZLny5YDD8mwUwFKD/yPvKqyABi8zsNlNsD6qFVMm1apm0qx70sM3gYPybY6ZWXP0yVErksaR5iCvJCL2zqVFZmZNUODMSpc98oZNsWhm1ur6ZGolIm7rzYaYmTVTK45GqVa185GbmfVpNa1f2WIcyM3MgGAR6JFLWjIi3suzMWZmzTK/wKmValYIGibpQeDx9HxjSefk3jIzs14UqOqt1VQzcPJsYDfgVYCIuB/YMc9GmZn1tvYatlZTTWqlX0Q8Iy3wW6gtp/aYmTVFK/a0q1VNIH9O0jAgJC0GHA08lm+zzMx6Vyv2tKtVTSA/iiy9sibwIvDXVGZm1me09eUeeUS8BIzohbaYmTVNgVd6q2qFoIuoMA1BRIzKpUVmZk3Q3pd75GSplA4fBb4MPJdPc8zMmqOvTpoFQEQssKybpCuAibm1yMysCfr6xc5yHwNWaXRDzMyaqV19OLUi6TU+/KujHzAHOCHPRpmZ9bYi3xzTZSBXdhfQxmTrdAK0RxR5ZTszs8qKPGqly1v0U9AeHxFtaXMQN7M+qR1VvbWaauZamS7pM7m3xMysiaKGrdV0tWbn4hExH/gMMFnSk8DbgMg665v2UhvNzHLXyNSKpH+RLVjfBsyPiM3L9gs4C9gVeAc4NCKm1VtfVznye4FNgT3qPbmZWVHkMPxwx4h4pZN9uwDrpW1L4Dfpa126CuQCiIgn6z25mVlRtPVu6ntP4PJ03fFuSctLWi0iZtdzsq4C+UqSvtPZzog4s54KzcxaUS09ckmjgNJpSkZHxOiS5wHcKimAC8v2AazOgnfIP5/KGh7IFwOWgRa8RGtm1mC1BPIUmMuDc6ntImKmpJWBCZIejYg7etbCznUVyGdHxGl5VWxm1koauWRnRMxMX1+SNA4YBpQG8pnAGiXPh/Dh/To162r4oXviZrbIaNRSb5L6SxrQ8RgYDswoO+wG4GBltgLm1psfh6575DvVe1Izs6Jp4C36qwDj0vKYiwNjIuJmSUcCRMQFwHiyoYdPkA0//FpPKuw0kEfEnJ6c2MysSBo1jjwiniKb2qS8/IKSxwF8qzE11jf7oZlZn7OoTWNrZtbnOJCbmRVcK86hUi0HcjMzij2NrQO5mRl9eGEJM7NFRXuBkysO5GZm+GKnmVnhFbc/7kBuZga4R25mVnjzVdw+uQO5mRlOrZiZFZ5TK2ZmBefhh2ZmBVfcMO5AbmYGOLViZlZ4bQXukzuQm5nhHrmZWeGFe+RmZsVW5B55v2Y3wLr2xeE78NCMO3j04Yn893ENW+LPCuyMc05j2j9vZ8Jdf2p2U/qUdqLqrdU4kLewfv36cfZZp7Pb7gey0cY7st9+e/GJT6zX7GZZk/1hzPUc/NWjmt2MPidq2FqNA3kLG7bFZ3jyyX/x9NPPMm/ePMaOvZ49dv9is5tlTXbvpKm8/trcZjejz5lPVL21GgfyFjZ49VV57vlZHzx/fuZsBg9etYktMuu7ooZ/rSbXi52SlgS+AgwtrSsiTuvk+FHAKAAtthz9+vXPs3lmZh8o8sXOvEetXA/MBaYC73V3cESMBkYDLP6R1Vvv114vmzXzBdYYMviD50NWX41Zs15oYovM+q5W7GlXK+9APiQids65jj5r8pTprLvuxxg6dA1mznyBfffdk4MO9sgVszwUuUeed478/yRtlHMdfVZbWxvHHHsS428aw4wHbuePf/wzDz/8WLObZU12zkU/57pbrmTtdYdyz4y/st+BX252k/qEtoiqt1ajyLFRkh4G1gWeJkutCIiI+HR3r3VqxSoZvMzAZjfBWtCzcx5UT8+x/1pfrjrmjHlmXI/ra6S8Uyu75Hx+M7OGKHKOPNfUSkQ8A6wBfC49fifvOs3M6tFew9Zq8h5+eAqwObABcCmwBHAlsG2e9ZqZ1aoVb72vVt6plS8DnwGmAUTELEkDcq7TzKxmRU6t5B3I34+IkBQAknyHj5m1pFYcjVKtvPPVYyVdCCwv6Qjgr8BFOddpZlazRs1+KGkNSX+X9LCkhyQdU+GYHSTNlTQ9bSf3pO259sgj4heSvgC8QZYnPzkiJuRZp5lZPRp4EXM+8N2ImJZSyVMlTYiIh8uOuzMidmtEhbkvLJECt4O3mbW0RuXII2I2MDs9flPSI8DqQHkgb5hcUyuS3pT0Rtn2nKRxktbOs24zs1rUklqRNErSlJJtVKVzShpKNuDjngq7t5Z0v6S/SPpUT9qed4/818DzwBiyuzpHAOuQjWK5BNgh5/rNzKpSy13upRP8dUbSMsC1wLER8UbZ7mnAWhHxlqRdgeuAuleNyfti5x4RcWFEvBkRb6Q3/8WIuAZYIee6zcyq1kZUvXVH0hJkQfyqiFhoTb4UD99Kj8cDS0hasd625x3I35G0r6R+adsX+HfaV9yxPmbW5zRw1IqAi4FHIuLMTo5ZNR2HpGFksfjVetued2rlAOAs4Pz0fBJwoKSlgG/nXLeZWdUaOIHgtsBBwIOSpqeyHwBrpnouAPYBjpI0H3gXGBE9aEDeww+fAnbvZPfEPOs2M6tFo27Rj4iJZNcEuzrmXODchlRIL05gJWlab9VlZlYrr9lZnZaav9fMrFSRb9HvzUB+Uy/WZWZWkyLPftgrqRVJg8huU92sN+ozM6tVo0atNEMugVzSjZI2TI9XA2YAhwFXSDo2jzrNzHoiIqreWk1ePfKPRcSM9PhrwISI2B3Ykiygm5m1FPfIFzav5PFOwHjIJpChNVdKMrNFnEetLOw5SUeTzbOyKXAzQLoRaImc6jQzq1tbFLePmVeP/HDgU8ChwH4R8Xoq34ps7U4zs5ZS5Bx5Xj3yN8gWkXiprHwG2W36ZmYtpRVz39XKq0d+NrBdhfLtgF/lVKeZWd2KnCPPK5Bv1snUjeOA7XOq08ysbu0RVW+tJq/UytJd7Ou1+V3MzKrVij3tauUVyF+SNCwi7i0tlLQF8HJOdZqZ1a3Io1byCuTHAWMlXQZMTWWbAweTLfdmZtZSWjFlUq1c0hypJz6MbMbDQ9MmYMuIqLQIqZlZUxX5Ymdusx+moYenlJZJWkPScRFxRl71mpnVwz3yLkhaSdI3Jd0J3A6sknedZma1co+8jKQBwN7A/sD6wJ/IJtIakkd9ZmY91RZtzW5C3XIbtQLcC5wETIyIkPTlnOoyM+uxVrz1vlp5pVa+DywJnA98X9I6OdVjZtYQnsa2TET8OiK2AvZMRdcBgyUdL2n9POo0M+uJIk+alevFzoh4KiJ+EhEbkY0jX5Y0N7mZWSsp8i36vXm7/NCIODEi1u3FOs3MqlLkUSu9GchP68W6zMxq0hbtVW+tJrcbgipQL9ZlZlaTVsx9VyuXHrmkrSoUfyOPuszMGsE58oWdX15QPhOimVkrKfKold5MrZiZtaxWHB9erbwC+dqSbuhsZ0TskVO9ZmZ1acWedrXyCuQvA7/M6dxmZg3XiqNRqpVXIH8zIv6R07nNzBquFS9iViuvQP6vnM5rZpaLIqdW8pprZW9JK0s6VdIf03aqJM9FbmYtqZF3dkraWdI/JT0h6YQK+5eUdE3af4+koT1pe17jyLcFJqenl6cN4J60z8yspTRq+KGkxYDzgF2ATwIjJX2y7LDDgdfSlCW/An7ek7bnlVr5JbBXRNxXUnaDpHHAhcCWOdVrZlaXBubIhwFPRMRTAJJ+TzYT7MMlx+wJ/DA9/iNwriRFnfmdvAL5smVBHICImJ5WD+rW/Pdn+pb+RNKoiBjd7HZYa/HnorFqiTmSRgGjSopGl/wsVgeeK9n3PAt3Xj84JiLmS5oLDAJeqbXdkN+dnZK0QoXCgTnW2ZeN6v4QWwT5c9EkETE6IjYv2Zr6CzWvoPor4FZJn5U0IG07AH9J+8zM+qqZwBolz4eksorHSFocWA54td4Kc0mtRMRoSbOAHwGfSsUPAT+OiD/nUaeZWYuYDKwn6WNkAXsE2UL0pW4ADgEmAfsAf6s3Pw45zrUSETcCN+Z1/kWM86BWiT8XLSjlvL8N3AIsBlwSEQ9JOg2YEhE3ABcDV0h6AphDFuzrpjwGwUs6uYvdERE/anilZmaLqLwC+XcrFPcnGzs5KCKWaXilZmaLqFwC+QIVZMMNjyEL4mOBX0bES7lWama2CMltKKCkgZJ+DDxAlovfNCKOX1SCuKS9JIWkj5eUrS9pvKTHJU2TNFbSKpKWlnSVpAclzZA0UZL/amlh6Wf7y5Ln35P0w5LnoyQ9mrZ7JW2XysdJmp5uzZ6bHk+XtE0n9Rwr6d+SlisrHybpjnQb+H2Sfps+R6tIulHS/ZIeljQ+p2+BtZBcLnZKOgPYm+xizEYR8VYe9bS4kcDE9PUUSR8FbgK+0zFyJw3JXAk4DHgxIjZK5RsA85rRaKvae8Dekn4aEQvcxCFpN7KlDbeLiFckbQpcJ2lYRHw5HbMD8L2I2K2bekaSjYLYG7g0vXYV4A/AiIiYlMr2AQaQLXI+ISLOSuWfbsi7tZaWV4/8u8Bg4CRglqQ30vampDdyqrNlpN70dmTppI6r0fsDk0qHX0bE7RExA1iNknGmEfHPiHivwnlXkjRB0kOpB/aMpBXTvgNTz2+6pAvTfA9IekvS6amHdrcnLmuY+WQdlf+qsO944LiOAB8R04DfAd+qpQJJ6wDLkP0/Glmy61vA7zqCeKrjjxHxItln6fmS8gc6O3f6PDwo6ceS3irZd5ykyZIekHRqKhsq6RFJF6XP362Slqrl/Vh+8pr9sF9ELBURAyJi2ZJtQEQsm0edLWZP4OaIeAx4VdJmwIbA1E6OvwQ4XtKk9J9qvU6OO4VsvOmnyOZnWBNA0ieA/YBtI2IToA04IL2mP3B3RGwM3AEc0fO3Z8l5wAHlaQ+yeyfKf9ZT+PCeimqNAH4P3AlsUPJLuKvP0nnAxZL+LulESYM7Oe4s4Kz0V+AHgV/ScGA9svlCNgE2k7R92r0ecF76/L0OfKXG92M58e3y+RhJ9h+Q9HVkF8cSEdOBtYEzgIHA5BScy23Xcd6IuBl4LZXvBGyWXjc9PV877XufD8fzTwWG1v52rJKIeINsZs//zKmKkcDvI6IduBb4ahVtuoXsZ38R8HHgPkkrVTh0a7L0DMCYkvLhabsPmJbO0dGxeDp9VsGfpZbixZcbLM0n8zlgI0lBdkNAAKcCn+3sdek6wp+AP0lqB3aV9Dk+7EHv2lW1ZH9qf7/Cvnkld4y14Z95o/2aLOBdWlL2MNkv1r+VlG1GdndzRZK2JJsZFOBk4GmyADpBEsBHUtm56TybAddXOldEzCELzmNtITt+AAAE/klEQVQk3Qhsn/L0X0r7N+ni/Qj4aURcuEBhNl92abqvDXBqpUW4R954+wBXRMRaETE0ItYg+w/4BLCNpC91HChpe0kbStpWaZIxSR8hm8P4mYg4LyI2Sdss4C5g33TccKBjYrLbgH0krZz2DZS0Vi+930VaCppjya6HdPhf4OeSBgFI2gQ4FDi/i/PcU/KzvoGsN/7D9BkaGhGDgcHp53oucEgK/qQ69k4jVj4naelUNgBYB3g2Ik7sOH96yd18mBopvavwFuCwjlFTklbv+FxZ63Igb7yRwLiysmvJ/rPsBhytbPjhw8A3yRaqXgf4h6QHyf6knZJeU+5UYLikGWR/Zr9Atj7qw2QXxG6V9AAwgeyil/WOXwIrdjxJgfgS4P8kPUqW5jgwImbXcM4RLPw5Gkc2UuXFtP8XafjhI8AXgTfJeupT0udgEvDbiJjMwo4FvpOOWxeYm9p+K1lvflL6PP6RbDSMtbDcbwiyxpG0JNCW5nLYGvhNN38mm1WUeu3vRkRIGgGMjIg9m90uq4/zpcWyJjBWUj+yi5gegWL12oy0Kg3ZCJTDmtwe6wH3yM3MCs45cjOzgnMgNzMrOAdyM7OCcyC3hUhqS3O2zJD0h45xyXWea4d0UwqS9pB0QhfHLi/pm3XU8UNJ36u2vOyYy9KEU9XWNTQN/zRrGQ7kVsm76eaRDclGxxxZulOZmj87EXFDRPysi0OWJxtbb2Y1cCC37twJrJt6ov+UdDkwA1hD0vA00de01HPvuBtwZ2XzcE8jm36VVH6opHPT41WUzc19f9q2AX4GrJP+GjgjHbfQTHyp/ERJj0maCGzQ3ZuQdEQ6z/2Sri37K+Pzkqak8+2Wjl9M0hkldX+jwjk/pQ9nnHxAnU92ZpYrB3LrlKTFgV2AB1PResD5afa7t8nuJv18RGxKdjfqd5TNu34RsDvZWOVVOzn92cA/0qyMm5LNH3IC8GT6a+C4zmbiUzab5IhUtiuwRRVv508RsUWq7xEWvKV+aKrjS8AF6T0cDsyNiC3S+Y9Qtip6qSPJZhDcBNicklkEzXqTbwiySpZKsyhC1iO/mGx++Wci4u5UvhXZnDB3lUzqNIlstrynI+JxAElXAqMq1PE54GCAiGgD5nbMN1OidCY+yObmXo/slvFxEfFOquOGKt7ThspWrFo+neeWkn1j0wyDj0t6Kr2H4cCnS/Lny6W6Hyt53STgRElDyH5RPF5FO8wazoHcKnm3/Nb/FKzfLi0iW4lmZNlxjZwyoLOZ+I6t41yXAXtFxP2SDgV2KNlXfldcpLqPTtPCltY99IODIsZIuoesJz9e0jcionTGQ7Ne4dSK1etuYFtJ6wJI6i9pfeBRYKiy1W2g87nYbwOOSq9dTNniDG+y4ARNnc3Edwewl6Sl0gx/u1fR3gHAbElL8OGiGx2+KqlfavPawD9T3Uel4zvWW+1f+iJJawNPRcTZZFPKelk1awr3yK0uEfFy6tlenSbzAjgpIh6TNAq4SdI7ZKmZSrPnHQOMlnQ42dzWR0XEJEl3peF9f0l58k+QzcQH8BbZLILTJF0D3A+8RLamZXf+B7iHbLbJe8ra9CxwL7AscGRE/FvSb8ly59PSfCQvA3uVnXNf4CBJ88hmovxJFe0wazjPtWJmVnBOrZiZFZwDuZlZwTmQm5kVnAO5mVnBOZCbmRWcA7mZWcE5kJuZFdz/A41mvwgYdpXoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"ACS\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20 , random_state=1)\n",
    "    \n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "\n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['ACS-gen','NOT-ACS-gen']); ax.yaxis.set_ticklabels(['ACS-gen' , 'NOT-ACS-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "    auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "    print('ROC AUC: %f' % auc)\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: 0.423077\n",
      "ROC AUC: 0.269231\n",
      "Accuracy: 0.866667\n",
      "Precision: 0.500000\n",
      "Recall: 0.500000\n",
      "F1 score: 0.500000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEWCAYAAABLzQ1kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xm8XdP9//HXO4khMhFUiSH4GooaozXU3KpWik6qVS3VxtCBKv1q+Zmq2mrjq2pqRBFTUaKKIvWlSgkRMQTlW6SGEAQ1J/fez++Pva6eXHfY59yzz9n33vczj/245+y9z1rr3pz7ueusvdZnKyIwM7PyGdTsBpiZWeccoM3MSsoB2syspBygzcxKygHazKykHKDNzErKAdp6TdJQSX+S9JqkK3pRzt6Sbqpn25pB0p8lfb3Z7bC+zwF6AJH0FUkzJL0haW4KJB+rQ9FfAFYAlo2IL9ZaSERcHBE716E9i5C0vaSQNLXD/o3S/ltzlnOcpIt6Oi8iPhURF9TYXLP3OEAPEJIOA04FTiILpqsCZwK716H41YDHIqKlDmUV5UVgS0nLVuz7OvBYvSpQxr9TVjd+Mw0AkkYBJwDfjoirIuLNiFgYEX+KiCPSOUtIOlXSc2k7VdIS6dj2kp6R9ANJ81Lve7907HjgGOBLqWe+f8eepqSxqac6JD3fV9ITkl6X9KSkvSv2317xuq0k3ZOGTu6RtFXFsVsl/UTSHamcmyQt182PYQFwNbBXev1g4EvAxR1+Vr+W9LSkf0u6V9I2af8uwI8rvs/7K9rxU0l3AG8Ba6R930zHz5J0ZUX5v5B0syTl/g+0AcsBemDYElgSmNrNOUcBWwAbAxsBHwGOrjj+QWAUMAbYHzhD0jIRcSxZr/yyiBgeEed21xBJw4DTgE9FxAhgK2BWJ+eNBq5L5y4LnAJc16EH/BVgP+ADwOLA4d3VDUwBvpYefxJ4CHiuwzn3kP0MRgOXAFdIWjIibujwfW5U8Zp9gAnACGBOh/J+AHw4/fHZhuxn9/VwjgXLwQF6YFgWeKmHIYi9gRMiYl5EvAgcTxZ42i1MxxdGxPXAG8A6NbanDdhA0tCImBsRszs5Z1fg8Yi4MCJaIuJS4FHgMxXnnBcRj0XE28DlZIG1SxHxd2C0pHXIAvWUTs65KCJeTnVOBJag5+/z/IiYnV6zsEN5b5H9HE8BLgK+GxHP9FCeGeAAPVC8DCzXPsTQhZVYtPc3J+17r4wOAf4tYHi1DYmIN8mGFg4E5kq6TtK6OdrT3qYxFc+fr6E9FwLfAXagk08Ukg6X9EgaVnmV7FNDd0MnAE93dzAipgNPACL7Q2KWiwP0wHAn8C6wRzfnPEd2sa/dqrz/439ebwJLVTz/YOXBiLgxIj4BrEjWKz4nR3va2/RsjW1qdyFwMHB96t2+Jw1B/BDYE1gmIpYGXiMLrABdDUt0O1wh6dtkPfHnUvlmuThADwAR8RrZhbwzJO0haSlJi0n6lKST02mXAkdLWj5dbDuG7CN5LWYB20paNV2g/FH7AUkrSNo9jUW/SzZU0tZJGdcDa6epgUMkfQlYD7i2xjYBEBFPAtuRjbl3NAJoIZvxMUTSMcDIiuMvAGOrmakhaW3gROCrZEMdP5TU7VCMWTsH6AEijaceRnbh70Wyj+XfIZvZAFkQmQE8ADwIzEz7aqlrGnBZKuteFg2qg1I7ngPmkwXLgzop42VgPNlFtpfJep7jI+KlWtrUoezbI6KzTwc3AjeQTb2bA7zDosMX7YtwXpY0s6d60pDSRcAvIuL+iHicbCbIhe0zZMy6I19MNjMrJ/egzcxKygHazKzOJP0uLep6qGLfLyU9KukBSVMlLd1TOQ7QZmb1dz6wS4d904ANImJDsuscP+r4oo4coM3M6iwibiO7CF6576aKtQR3ASv3VE53CxeaauFLT/jqpb3P0JW2aXYTrIRaFjzb69wm1cScxZdf8wCy5f3tJkXEpCqq+wbZTKdulTZAm5mVVQrG1QTk90g6imy+/cU9nesAbWYG0NZaeBWS9iWb379TnoRZDtBmZgCtxaYzTylrfwhs1zHNQFccoM3MgIjOMg7URtKlwPZkScqeAY4lm7WxBDAtpQO/KyIO7K4cB2gzM4C2+gXoiPhyJ7u7zZXeGQdoMzOAOvag68UB2swMGnKRsFoO0GZm4B60mVlZRcGzOGrhAG1mBnW9SFgvDtBmZuAhDjOz0vJFQjOzknIP2syspHyR0MyspHyR0MysnCI8Bm1mVk4egzYzKykPcZiZlZR70GZmJdW6sNkteB8HaDMz8BCHmVlpeYjDzKyk3IM2MyspB2gzs3IKXyQ0Myspj0GbmZWUhzjMzErKPWgzs5JyD9rMrKTcgzYzK6kWJ+w3Mysn96DNzErKY9BmZiXlHrSZWUm5B21mVlIl7EEPanYDzMxKoaUl/9YDSb+TNE/SQxX7RkuaJunx9HWZnspxgDYzA4jIv/XsfGCXDvuOBG6OiLWAm9PzbjlAm5lBNgadd+tBRNwGzO+we3fggvT4AmCPnsrxGLSZGVR1kVDSBGBCxa5JETGph5etEBFz0+PngRV6qscB2swMqrpImIJxTwG5u9eHpB7HShygzcwAWluLruEFSStGxFxJKwLzenqBx6DNzKCuY9BduAb4enr8deCPPb3APWgzM6jrQhVJlwLbA8tJegY4Fvg5cLmk/YE5wJ49leMAbWYGdV2oEhFf7uLQTtWU4wBtZgZEW675zQ3lAG1mBs7FYWZWWsXP4qiaA7SZGbgHbWZWWg7Q1pOjTzqF2+64m9HLLM3VF50NwK9On8xf75jOkMWGsMqYFTnxx4cxcsTwJrfUmuWcSRPZ9dMfZ96LL7HxJlVNCrDu5EuC1FBeqFIye3z6E5x9yomL7Nty802YeuHZTJ1yFmNXGcPkCy9rUuusDKZMuZxdx+/d7Gb0P8UvVKmaA3TJjNv4w4waOWKRfVt/dDOGDBkMwIbrr8sL815qRtOsJP52+3Tmv/Jqs5vR/7RF/q1BCh3ikLQ1cBywWqpLZHlC1iiy3v5s6nU3sctO2zW7GWb9TwlncRTdgz4XOAX4GLA5MC597ZSkCZJmSJoxecqlBTet7/ntBZcyePBgxu+8Q7ObYtbvRFtb7q1Rir5I+FpE/DnvyZUp/Ba+9ET5Ruyb6OrrpnHbHXcz+bSfIanZzTHrfwbgSsJbJP0SuAp4t31nRMwsuN5+5fa7ZvC7S67g/NNPZuiSSza7OWb9UwlvGlt0gP5o+jquYl8AOxZcb591xLE/5577HuDVV//NTnt8lYP334fJF17GgoUL+dahRwHZhcJjf/jdJrfUmuWiC89gu223ZLnlRvPUEzM4/oRfcd75v292s/q+EvagFSWc+wce4rDODV1pm2Y3wUqoZcGzvR73e/OYvXLHnGEn/L4h44yFXiSUtIKkcyX9OT1fL+VCNTMrl2jLvzVI0bM4zgduBFZKzx8DDi24TjOz6pVwHnTRAXq5iLgcaAOIiBagfJMNzWzAG4jT7N6UtCzZhUEkbQG8VnCdZmbVK+FFwqID9GFkN0pcU9IdwPLAFwqu08ysegMtQEfETEnbAeuQLfP+R0QsLLJOM7OalHCpd9G5OD7XYdfakl4DHoyIeUXWbWZWjYF4T8L9gS2BW9Lz7YF7gdUlnRARFxZcv5lZPgMwQA8BPhQRL0A2LxqYQrbC8DbAAdrMymEA3lFllfbgnMxL++ZL8li0mZXHAOxB3yrpWuCK9Pzzad8wwBnHzaw8BmCA/jbwObJ80JANb1wZWQIQJzU2s9KI1gE2xJEC8ZXAlZLGR8S1RdZnZlazEvagG3lPwhMaWJeZWVWiLXJvjVL0EEcl3wbEzMqrhD3oRgboAxpYl5lZdco3BF18gJa0AbAesKSkdQEiYkrR9ZqZVSNa6hehJX0f+CZZorgHgf0i4p1qyyk6Yf+xwG/StgNwMrBbkXWamdWkrYqtG5LGAN8DxkXEBsBgYK9amlT0RcIvADsBz0fEfsBGwKiC6zQzq1qdLxIOAYZKGgIsBTxXS5uKDtBvR0Qb0CJpJGklYcF1mplVr4oetKQJkmZUbBPai4mIZ4FfAf8C5gKvRcRNtTSp6DHoGZKWBs4hS5L0BnBnwXWamVWtmulzETEJmNTZMUnLALsDq5OtmL5C0lcj4qJq21T0QpWD08OzJd0AjIyIB4qs08ysJvW7Rvhx4MmIeBFA0lXAVkA5ArSkdSPiUUmbdnJsE2B+RMwpom4zs1pES92K+hewhaSlgLfJrsPNqKWgonrQhwETgIldHF9W0v0RsU9B9ZuZVSXq1IOOiOmS/gDMBFqA++hiOKQnPQbodFeUaRHxuqQjgU2BkyJiVjcNnCBpEHB0RNzRRbk1DZqbmRWijgtVIuJY4NjelpNnFsdxKThvBXwauBg4u6cXpdkbp3dzfOfcrTQzK1i05d8aJU+Abr+T4njgtxHxR2CJnOXfLOnzkpyHw8xKrYwBOs8Y9FxJZwC7AOMkLU7++dMHkI1Ht0h6hyxhUkTEyJpaa2ZWkGgtXz8yT4Dek2xo4zcR8YqklYAj8xQeESMkjQbWApasvZlmZsVqZM84ry4DdFr51+6Gin1vAJ1e+OukjG8ChwArA7OALYC/k007MTMrjWjrWz3o2WSZmCpb3f48gFVzlH8IsDlwV0TskLLZnVRjW83MCtOnetARUY+cGe9ExDuSkLREWryyTh3KNTOrq4i+1YN+j6S9gDUi4iRJKwMrRMS9OV76TMrFcTUwTdIrgFcQmlnp9KkedDtJpwOLAduSDU+8RTYPevOeXhsRn00Pj5N0C1mq0Rtqbq2ZWUHa+ugsjq0iYlNJ9wFExPw01a4qEfHXqltnZtYgfe0iYbuFadl2AEhallLevcvMrHZlDNB5FpycAVwJLC/peOB24BeFtsrMrMEi8m+N0mMPOiKmSLqXLMcpwBcj4qFim2Vm1lhl7EHnTTc6GFhINsxR9G2yzMwarozT7HoMtpKOAi4FViJbEXiJpB8V3TAzs0ZqbVXurVHy9KC/BmwSEW8BSPopWQLqnxXZMDOzRipjDzpXNrsO5w1J+8zM+o0+NQYt6X/IxpznA7Ml3Zie7wzc05jmmZk1RiNnZ+TVXQ+6fabGbOC6iv13FdccM7Pm6FM96Ig4t5ENMTNrpta28k1Qy5OLY03gp8B6VCTdj4i1C2yXmVlDlXGII8+fjPOB88jyQH8KuBy4rMA2mZk1XFso99YoeQL0UhFxI0BE/DMijiYL1GZm/UaEcm+Nkmea3bspWdI/JR0IPAuMKLZZZmaNVcYhjjwB+vvAMOB7ZGPRo4BvFNkogKErbVN0FWZm72nk0EVeeZIlTU8PXwf2KbY5ZmbN0admcUiaSsoB3ZmI+FwhLTIza4ISjnB024M+vWGtMDNrsj41xBERNzeyIWZmzdRXkyWZmfV7ZbyPnwO0mRkQlK8HnfuypaQlimyImVkztYRybz2RtLSkP0h6VNIjkraspU157qjyEUkPAo+n5xtJ+k0tlZmZlVWg3FsOvwZuiIh1gY2AR2ppU54e9GnAeOBlgIi4H9ihlsrMzMqqrYqtO5JGAdsC5wJExIKIeLWWNuUJ0IMiYk6Hfa21VGZmVlbV9KAlTZA0o2KbUFHU6sCLwHmS7pM0WdKwWtqUJ0A/LekjQEgaLOlQ4LFaKjMzK6tqetARMSkixlVskyqKGgJsCpwVEZsAbwJH1tKmPAH6IOAwYFXgBWCLtM/MrN9oRbm3HjwDPFORJuMPZAG7anlyccwD9qqlcDOzvqJed7yKiOclPS1pnYj4B7AT8HAtZeW5o8o5dLJMPSImdHK6mVmf1FbfedDfBS6WtDjwBLBfLYXkWajyl4rHSwKfBZ6upTIzs7KqZ7KkiJgFjOttOXmGOBa5vZWkC4Hbe1uxmVmZ9Jel3qsDK9S7IWZmzdSm8i31zjMG/Qr/6f0PAuZT45QRM7OyKuPijm4DtCSRLVN8Nu1qiyjjnbvMzHqnXrM46qnbedApGF8fEa1pc3A2s36pDeXeGiXPQpVZkjYpvCVmZk0UVWyN0t09CYdERAuwCXCPpH+SLVkUWee6ppUxZmZlVMYhju7GoO8mW564W4PaYmbWNH1tmp0AIuKfDWqLmVnTtPaxHvTykg7r6mBEnFJAe8zMmqKv9aAHA8OhhDfqMjOrs74WoOdGxAkNa4mZWRPluNVgw/U4Bm1mNhD0tR70Tg1rhZlZk/Wppd4RMb+RDTEza6a+Ng/azGzA6GtDHGZmA4YDtJlZSZUxE5wDtJkZHoM2MyutPjWLw8xsIGkr4SCHA7SZGb5IaGZWWuXrPztAm5kB7kGbmZVWi8rXh3aANjPDQxxmZqXlIQ4zs5LyNDszs5IqX3h2gDYzAzzEYWZWWq117kNLGgzMAJ6NiPG1lOEAbWZGIT3oQ4BHgJG1FjCofm0xM+u7oop/PZG0MrArMLk3bXKANjMj60Hn3SRNkDSjYpvQobhTgR/Sy465A3SJnTNpIs89cz+z7ru52U2xEvH7ohhtRO4tIiZFxLiKbVJ7OZLGA/Mi4t7etskBusSmTLmcXcfv3exmWMn4fVGMqGLrwdbAbpKeAn4P7Cjpolra5ABdYn+7fTrzX3m12c2wkvH7ohgtRO6tOxHxo4hYOSLGAnsB/xsRX62lTZ7FYWYGuS7+NVqhAVrS2sARwGqVdUXEjl2cPwGYAKDBoxg0aFiRzTMze08RC1Ui4lbg1lpfX3QP+grgbOAcctzyKw20TwIYsviY8v05M7N+a8D1oIGWiDir4DrMzHqtjEu9i75I+CdJB0taUdLo9q3gOvuNiy48g9tvu4Z11l6Tp56YwX777tXsJlkJ+H1RjNaI3FujKAqsTNKTneyOiFijp9d6iMPM8mpZ8Kx6W8ZXVvts7phzyZypva4vj0KHOCJi9SLLNzOrlzKOQRc6xCFpKUlHS5qUnq+VVtmYmZVKNUu9G6XoMejzgAXAVun5s8CJBddpZla1apZ6N0rRAXrNiDgZWAgQEW8BDRm7MTOrRj2z2dVL0dPsFkgaSlq+LmlN4N2C6zQzq1ojZ2fkVXSAPha4AVhF0sVkSUT2LbhOM7OqDbibxkbENEkzgS3IhjYOiYiXiqzTzKwWZVyoUnQujk3Tw7np66qSRgFzIqKlyLrNzKpRxml2RQ9xnAlsCjxA1oPeAJgNjJJ0UETcVHD9Zma5lHGIo+hZHM8Bm6Q7DmwGbAI8AXwCOLngus3McouI3FujFN2DXjsiZrc/iYiHJa0bEU9Inm1nZuXRWsIedNEBeraks8hu+wLwJeBhSUuQ5kabmZVBGYc4ig7Q+wIHA4em53cAh5MF5x0KrtvMLLdGDl3kVfQ0u7eBicBESZtGxMyKw28UWbeZWTXK2INu5E1jJzewLjOzqgzEpd6VfFXQzEprIC71rnR8A+syM6tKGYc4Cg/QkpYB1gLmS9oWICJuK7peM7NqDLgALembwCHAysAsspwcdwI7FlmvmVm1yjiLo+iLhIcAm5Pl3tiBbCXhqwXXaWZWtTIm7C96iOOdiHhHEpKWiIhHJa1TcJ1mZlUbiMmSnpG0NHA1ME3SK8Ccgus0M6taa5Qv4WjRC1U+mx4eJ+kWYBRZAn8zs1Ip4xh0IQFa0siI+Lek0RW7H0xfh0lqjYjWIuo2M6vFQJrFcQkwHriX7H6E7YtU2h8Pl3RORPy4oPrNzKoyYMagI2K8snyi20XEvzoelzQYeAhwgDazUmgr4RBHYdPsIhvQua6LY60R8aGi6jYzq1YZc3EUPQ96pqTNC67DzKzXWqMt99YdSatIukXSw5JmSzqk1jYVPc3uo8DekuYAb5KNP0dEbFhwvWZmVanjEEcL8IOImClpBHCvpGkR8XC1BRUdoD9ZcPlmZnVRr6GLiJgLzE2PX5f0CDAGKFeAjog5AJI+ACxZZF1mZr1RTQ9a0gRgQsWuSRExqZPzxpKluJheS5uKTpa0G9kdVVYC5gGrAY8A6xdZr5lZtarpQadg/L6AXEnScOBK4NCI+HctbSr6IuFPyDLYPRYRqwM7AXcVXKeZWdVaozX31hNJi5EF54sj4qpa21R0gF4YES8DgyQNiohbgHEF12lmVrWIyL11J60BORd4JCJO6U2bir5I+Grq5t8GXCxpHtlsDjOzUqnjUu+tgX2AByXNSvt+HBHXV1uQikwQImkY8DZZT31vsmRJF6dedbeGLD6mfMt6zKyUWhY82+t7no5ZZv3cMefZV2Y35B6rRc/iaO8tt0l6OSIuKLI+M7NaDail3p04oYF1mZlVpYxLvRt5V++GfCQwM6tFGRP2F9KDlrRFJ7sPKKIuM7N6qNcsjnoqaojjzI47IuLuguoyM+u1tojcW6M0cojDzKy0Bswtr4A1JF3T1cGI2K2ges3MajKQbnn1IlkODjOzPmEg9aBfj4i/FlS2mVndlXEWR1EB+qmCyjUzK0QZF6oUddPYz6Uc0N/mP6lFZwNnRsQLRdRpZtYbZRziKGoe9NbAPenplLQBTE/HzMxKZSCtJJwI7BER91Xsu0bSVOC3ZPcqNDMrjTL2oIsK0CM7BGcAImJWuomimVmpDJgxaLKc1ctExCsddo4m57BKPdIH9heSJnR2vzMb2Py+qK8yxpyilnr/D3CTpO0kjUjb9sCf0zGrzoSeT7EByO+Lfq6oWRyTJD1Hdk/CylkcJ0bEn4qo08ysvyksF0dEXAtcW1T5Zmb9XSEBWtIx3RyOiPhJEfX2Yx5ntM74fdHPFXJPQkk/6GT3MGB/YNmIGF73Ss3M+plCbxoLkKbVHUIWnC8HJkbEvEIrNTPrBwobg05T6g4ju5v3BcCmHafdmZlZ14pa6v1LsqXerwMfjojjHJzfT9JkSes1ux1WG0khaWLF88MlHVfxfIKkR9N2t6SPpf1TJc2S9H+SXkuPZ0naqhdtuV7S0r36hqx0ihqDbgPeBVpgkYXrIrtIOLLulfYxkgZHRGuz22G1k/QOMBfYPCJeknQ4MDwijpM0Hjge+GQ6tilwNfCRiHg+vX574PCIGN+LNojs97h8uTKt1wrpQUfEoIgYGhEjImJkxTZioARnSVdLulfSbEkT0r43JE2UdD+wpaRbJY1Lx86SNCOdf3wXZQ6SdGbqkU1LvaYvpGObSfprqvNGSSum/bdK+kXqwT0maZsG/QgGghaymRTf7+TYfwNHRMRLABExk2yo79t5C5c0XNLNkmZKelDS7mn/WEn/kDQFeAhYRdJTkpZLx9/33uuk7KUkXS7p4dSjn17xXtxZ0p2p3iskDU/7n5J0fEV71s37vViNqrmTrbeq7vo7On0dSvZLtCzZp4k9K865FRjX4fzBaf+GnZT5BeB6sj+sHwReSfsWA/4OLJ/O+xLwu4o6JqbHnwb+0uyfTX/ZgDeAkWT5z0cBhwPHpWPzgVEdzt8duKri+fbAtd2UP4Qsrw3AcsD/kX0KHQu0AVtUnPsUsFxX771Oyj4c+G16vAHZH5txqZ7bgGHp2H8Dx1TU8d30+GBgcrP/D/r75pvGFud7kj6bHq8CrAW0Ald2cf6eqbczBFgRWA94oMM5HwOuiOzj7POSbkn71yH7JZuWfeJlMNlH73ZXpa/3kv1yW51ExL9TT/Z7wNt1Ll7ASZK2JQvIY4AV0rE5EXFXF6/r7L33codzPgb8GiAiHpLU/l7bguy9d0d6Ly0O3Fnxusr30udq+aYsPwfoAqSxxY8DW0bEW5JuBZYE3olOxp0lrU7Wo9k8Il6RdD6wpKSPkqVnBehu8Y+A2RGxZRfH301fW/H/eRFOBWYC51XsexjYDPjfin2bkaU86FQn/9+jgeWBzSJioaSnyN5HAG92Ucb2dPLeSwH72HTaN7v5XgRMi4gvd3Hc76UGKipZ0kA3Cngl/YKsS9Yr6c5Isl+41yStAHwKICKmR8TGabsGuAP4fBqLXoHsIzLAP4DlJW0JIGkxSeu/rxYrRETMJ5vjv3/F7pOBX0haFkDSxsC+wJndlNPx/3sUMC8F5x2A1XI0p9P3XkRMrSh7Btl7ac/UtvWAD6fX3wVsLem/0rFhktbO95OwevNfwGLcABwo6RGy4NnVR1HIZrXcL+k+4FHgabJfns5cCexE1jt7mqzX9lpELEgXC0+TNIrs//VUuumtWd1NBL7T/iQirpE0Bvi7pCCbcvrViJjbVQGduBj4k6QHgRlk74/uBPnfe2cCF0h6OJU7m+y99KKkfYFLJS2Rzj0aeKyKdludFL6S0LqWfvF2i4gnq3jN8Ih4I/XM7ga2jjRtywYmSYOBecAHI2JhFa9ZLCLekbQm8BdgnYhYUGBTrUruQTeJpGnAg9UE5+TatCBhceAnDs5G1vudnDc4J0sBt0hajGzc+WAH5/JxD9rMrKR8kdDMrKQcoM3MSsoB2syspByg7X0ktSrLrvZQysWwVC/K2l7StenxbpKO7ObcpSUdXEMdxylLVJRrf4dzzm/PZ5KzrrGSHqq2jWa1cIC2zrydFjRsACwADqw8qEzV752IuCYift7NKUuT5XgwMxygrWd/A/6riwxqXWU920VZxr2ZVORrkLSvpNPT4xVSFrX707YV8HNgzdR7/2U67whJ90h6QBVZ/iQdpSw73+1kuUi6JelbqZz7JV3Z4VPBx5VlEnxMWZpQJA2W9MuKug/opMz1lWUJnJXOWav6H69Z1xygrUuShpAtO38w7VoLODMi1idbmn408PGI2JRspdthkpYEzgE+Q5Z74oNdFH8a8NeI2AjYlGwu75HAP1Pv/QhJO6c6PwJsDGwmaVtJmwF7pX2fBjbP8e1cFRGbp/oeYdFl2WNTHbsCZ6fvYX+ylXWbp/K/lXKmVDoQ+HVEbEyWCe6ZHO0wy80LVawzQyXNSo//BpwLrMSiGdS6ynq2LvBkRDwOIOkioLOcxDsCXwNICaRek7RMh3N2Ttt96flwsoA9ApgaEW+lOq7J8T1tIOlEsmGU4cCNFccuTxkCH5f0RPoedgY2rBifHpXqrlzyfCdwlKSVyf4APJ6jHWa5OUBbZ95OvcL3pCBcmUGt06xnKSlQvQj4WUT8dpGd0qE1lHU+sEfKe7Iv/0k0BYve9af9uchyH1cGciSSF1F2AAABJklEQVSNfe+kiEskTSfreV8v6YCIqMxeZ9YrHuKwWnWV9exRYGzK7wDQVdrKm4GD0msHpyRPr5P1jtvdCHyjYmx7jKQPkCWU30PSUGV3jf9MjvaOAOampc17dzj2RWUZAtcE1iBLMnQjcFA6H0lrSxpW+SJJawBPRMRpwB+BDXO0wyw396CtJl1lPYuIx5TdeOA6SW+RDZGM6KSIQ4BJkvYnyy18UETcKemONI3tz2kc+kPAnakH/wZZRriZki4D7idLEnRPjib/P2A68GL6Wtmmf5ElnhoJHJgSCE0mG5ueqazyF4E9OpS5J7CPpIXA88BJOdphlptzcZiZlZSHOMzMSsoB2syspBygzcxKygHazKykHKDNzErKAdrMrKQcoM3MSur/A/qVFYfTw8Q4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"arja\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20,random_state=1)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "    \n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pre\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['arja-gen','NOT-arja-gen']); ax.yaxis.set_ticklabels(['arja-gen' , 'NOT-arja-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "    auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "    print('ROC AUC: %f' % auc)\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: 0.000000\n",
      "CapGen label is not good\n",
      "Accuracy: 0.933333\n",
      "Precision: 0.000000\n",
      "Recall: 0.000000\n",
      "F1 score: 0.000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/macbook/miniconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 due to no true samples.\n",
      "  'recall', 'true', average, warn_for)\n",
      "/Users/macbook/miniconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEWCAYAAAB7QRxFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcXEW9/vHPE/Yl7JtsRlYXBJSIbCKoICCbC4gCIqARRFwQvCBcBASu+rsqICAGQUABQb0RBBQQF3bIYhASkH3JIrsQ9mTm+/ujqkmn0zPT3dNnus/kefM6L/rUOaeqZqbz7eo6VXUUEZiZWXmN6HQFzMxscBzIzcxKzoHczKzkHMjNzErOgdzMrOQcyM3MSs6B3AZN0hKS/iDpBUm/GUQ++0q6rp116wRJf5R0QKfrYQsOB/IFiKTPSpog6SVJM3PA2aYNWX8KWBVYMSL2ajWTiLg4InZsQ33mIWk7SSFpXE36Jjn9bw3mc4KkXw10XkTsHBEXtlhds6Y5kC8gJB0BnAacSgq6awNnA3u0Ifu3AvdHxJw25FWUp4EtJa1YlXYAcH+7ClDif1M25PymWwBIWhY4CTgsIv4vIl6OiNkR8YeIOCqfs5ik0yTNyNtpkhbLx7aTNE3SNyU9lVvzB+ZjJwLHA5/OLf2Da1uukkbllu/Cef/zkh6WNEvSI5L2rUq/ueq6rSSNz1024yVtVXXsb5K+K+mWnM91klbq59fwBvB7YJ98/ULAp4GLa35Xp0t6QtKLkiZK+kBO3wn4dtXPeVdVPU6RdAvwCrBOTvtCPv5TSb+ryv/7km6QpIb/gGYDcCBfMGwJLA6M6+ecY4EtgE2BTYDNgeOqjq8GLAusARwMnCVp+Yj4DqmVf1lELB0R5/VXEUlLAWcAO0fESGArYHKd81YArs7nrgj8CLi6pkX9WeBAYBVgUeDI/soGLgI+l19/FLgHmFFzznjS72AF4BLgN5IWj4g/1fycm1Rdsz8wBhgJPFaT3zeBd+cPqQ+QfncHhNfGsDZyIF8wrAg8M0DXx77ASRHxVEQ8DZxIClAVs/Px2RFxDfASsGGL9ekFNpK0RETMjIgpdc75GPBARPwyIuZExKXAfcBuVef8IiLuj4hXgctJAbhPEXErsIKkDUkB/aI65/wqIp7NZf4QWIyBf84LImJKvmZ2TX6vkH6PPwJ+BRweEdMGyM+sKQ7kC4ZngZUqXRt9WJ15W5OP5bQ386j5IHgFWLrZikTEy6QujUOAmZKulvT2BupTqdMaVfv/bqE+vwS+AmxPnW8oko6UdG/uzvkP6VtIf102AE/0dzAi7gAeBkT6wDFrKwfyBcNtwOvAnv2cM4N007JibebvdmjUy8CSVfurVR+MiGsjYgfgLaRW9rkN1KdSp+kt1qnil8CXgWtya/lNuevjW8DewPIRsRzwAikAA/TVHdJvN4mkw0gt+xk5f7O2ciBfAETEC6QbkmdJ2lPSkpIWkbSzpB/k0y4FjpO0cr5peDypK6AVk4FtJa2db7QeUzkgaVVJe+S+8tdJXTS9dfK4BtggD5lcWNKngXcCV7VYJwAi4hHgg6R7ArVGAnNII1wWlnQ8sEzV8SeBUc2MTJG0AXAysB+pi+VbkvrtAjJrlgP5AiL39x5BuoH5NKk74CukkRyQgs0E4J/A3cCknNZKWdcDl+W8JjJv8B2R6zEDeI4UVA+tk8ezwK6km4XPklqyu0bEM63UqSbvmyOi3reNa4E/kYYkPga8xrzdJpXJTs9KmjRQObkr61fA9yPiroh4gDTy5ZeVEUFm7SDfPDczKze3yM3MSs6B3Mys5BzIzcxKzoHczKzk+psg0lGzn3nYd2FtPm/bYPdOV8G60LTn7hn02jXNxJxFVlqnq9bKcYvczKzkurZFbmY2pHp7Ol2DljmQm5kB9HTzcvr9cyA3MwMi6q0UUQ4O5GZmAL0O5GZm5eYWuZlZyflmp5lZyblFbmZWbuFRK2ZmJeebnWZmJeeuFTOzkvPNTjOzknOL3Mys5Hyz08ys5Hyz08ys3CLcR25mVm7uIzczKzl3rZiZlZxb5GZmJdczu21ZSTof2BV4KiI2ymn/D9gNeAN4CDgwIv5T59pHgVlADzAnIkYPVJ6f2WlmBqlrpdFtYBcAO9WkXQ9sFBEbA/cDx/Rz/fYRsWkjQRwcyM3MkuhtfBsoq4gbgedq0q6LiMpg9duBNdtVdQdyMzNod4t8IAcBf+zjWADXSZooaUwjmbmP3MwMmgrQOcBWB9mxETG2wWuPBeYAF/dxyjYRMV3SKsD1ku7LLfw+OZCbmQHRxM3OHLQbCtzVJH2edBP0wxERfeQ9Pf//KUnjgM2BfgO5u1bMzKCtfeT1SNoJ+Bawe0S80sc5S0kaWXkN7AjcM1DeDuRmZtDWPnJJlwK3ARtKmibpYOBMYCSpu2SypHPyuatLuiZfuipws6S7gDuBqyPiTwOV564VMzNo64SgiPhMneTz+jh3BrBLfv0wsEmz5TmQm5mBp+ibmZWep+ibmZXcHD9Ywsys3NwiNzMrOfeRm5mVnFvkZmYl5xa5mVnJuUVuZlZyHrViZlZy9dewKgUHcjMzcB+5mVnpOZCbmZWcb3aamZVcT0+na9AyB3IzM3DXiplZ6TmQm5mVnPvIzczKLXo9jtzMrNzctWJmVnIetWJmVnJukZuZlVyJA/mITlfA5nXcqT9i24/tw577HTLfsQsu/R0bbb0zz//nhQ7UzLrF//7ku0z+19/58y3jOl2V4SWi8a3LOJB3mT132YFzfnTyfOkzn3yaW++cxFtWXaUDtbJu8ptLfs9+e83/QW+D1Nvb+NZlHMi7zOhN382yy4ycL/0HZ/yMI758MFIHKmVd5Y7bJvKf5/2trO16o/GtyxTeRy5pIWDV6rIi4vGiyx1O/nLTbayy8kq8ff11Ol0Vs+GrxKNWCm2RSzoceBK4Hrg6b1f1c/4YSRMkTfj5RZcWWbXSePW11zj3osv4yhf273RVzIa16O1teOs2RbfIvwZsGBHPNnJyRIwFxgLMfubh7vv+0gFPTJ/J9Bn/5pMHfBmAJ59+hr0OOpxfn3saK624QodrZzaMdGGXSaOKDuRPAO7MG4QN1n0bN1796zf3d/zkAVx23hksv9yyHayV2TBU4rVWir7Z+TDwN0nHSDqishVcZqkd9Z3vse+XvsGjj0/jw3vux+/+cG2nq2Rd5sxzf8AV117MuuuNYvw9f2af/T7R6SoNDyW+2akocEykpO/US4+IEwe61l0rVs/bNti901WwLjTtuXsGPZ7r5eP3aTjmLHXSr/stT9L5wK7AUxGxUU5bAbgMGAU8CuwdEc/XufYA4Li8e3JEXDhQfQrtWqkEbElLRsQrRZZlZjYo7e1auQA4E7ioKu1o4IaI+J6ko/P+f1VflIP9d4DRQAATJV1ZL+BXK3rUypaSpgL35f1NJJ1dZJlmZi1pY9dKRNwIPFeTvAdQaV1fCOxZ59KPAtdHxHM5eF8P7DRQeUX3kZ+WK/YsQETcBWxbcJlmZk1rZvhh9VDpvI1poIhVI2Jmfv1v0vyaWmuQBolUTMtp/Sp8QlBEPKF5pyOWd9S9mQ1fTdzErB4q3YqICEltuw9YdIv8CUlbASFpEUlHAvcWXKaZWfOKH7XypKS3AOT/P1XnnOnAWlX7a+a0fhUdyA8BDiN9NZgObJr3zcy6S09P41trrgQOyK8PAK6oc861wI6Slpe0PLBjTutX0aNWngH2LbIMM7N2aOczOyVdCmwHrCRpGmkkyveAyyUdDDwG7J3PHQ0cEhFfiIjnJH0XGJ+zOikiam+azqfQQC7pjDrJLwATIqLep5GZWWe0MZBHxGf6OPThOudOAL5QtX8+cH4z5RXdtbI4qTvlgbxtTOrzOVjSaQWXbWbWuBKvR170qJWNga0jogdA0k+Bm4BtgLsLLtvMrHFdOPW+UUUH8uWBpZm7cNZSwAoR0SPp9YLLNjNrnAN5n34ATJb0N0CkyUCnSloK+HPBZZuZNSx6uq/LpFFFj1o5T9I1wOY56dsRMSO/PqrIss3MmlLiFnnhz+yMiJl5hMp7qoK4mVlXid5oeOs2Q/nwZa8/ambdq8TrkRe+1koVP//dzLpXebvIhzSQbzaEZZmZNSXmlDeSFz2zc2Xgi6QnYixcWQUxIg4qslwzs6aVN44X3iK/gjQB6M94+Voz62LdeBOzUUUH8iUj4r8GPs3MrMNK3CIvetTKVZJ2KbgMM7NBK/Pww6Jb5F8Dvi3pDeAN0siViIhlCi7XzKw5JW6RFz2zc2SR+ZuZtUvM6XQNWldo14qS/ST9d95fS9LmA11nZjbUorfxrdsMGMglfULSyPz6aEmXS9q0wfzPBrYEPpv3XwLOaqmmZmZF6m1i6zKNtMhPiIhZ+SHKuwAXA+c0mP/7I+Iw4DWAiHgeWLSlmpqZFWhYt8iZO/57V+BneQGsxRrMf7akhYCANycIdeGvwcwWdGUO5I3c7Jwp6SxgJ2C0pEVpvG/9DGAcsIqkU4BPAce1VFMzswJFT3mXg2okkO9N6lL5SUQ8L2l14OhGMo+IiyVNJD1wVMCeEXFvy7U1MytIN7a0G9VnIJdUPdb7T1VpLwG3NFpARNwnaRawUM5j4YgyD/Qxs+Eoeodni3wKqW+7+qer7Aewdl8XSjoGWCQiTspJt5Ke27kocCHwP4Oos5lZ2w3LFnlErDWIfPcCPlC1/1xEvCff+Pw7DuRm1mUiytsib+impaR9JH07v15T0oBri0fEy1W7p+e0HmCJVipqZlakMo9aaWRC0JnA9sD+OekVBh5HvrSkRSo7EXFBzmsxwOusmFnX6e1Rw1u3aaRFvlVEfIm5k3qeY+BJPb8FfiZpyUqCpKVIHwC/bbGuZmaFiV41vHWbRgL5bEkjmDupZ0UGntTz38BTwOOSJkqaBDwKPJmPmZl1lTIH8kbGkZ8F/A5YWdKJpHHlJ/Z3Qe4LPzqfv15OfjAiXh1MZc3MihLdt8x4wwYM5BFxUZ7U85GctFdE3NNg/gHsAGwDhKSbgZ9GxGst1dbMrCDd2NJuVKNT7RcCZpMeDtHM0rcXAe8CfgKcCbwT+GUzFTQzGwoRanjrj6QNJU2u2l6U9PWac7aT9ELVOccPpu4DtsglHUtahnYcaTLQJZIujohGxoJvFBHvrNr/q6SprVXVzKw4PW0ajRIR/wI2BchzZ6aT4metmyJi13aU2Ugf+eeA90TEK7lipwD/oLFJPZMkbRERt+dr3w9MaLWyZmZFKWhC0IeBhyLisSIyr2ho9cOa8xbOaY3YDLhV0uN5f23gX5LuJj27c+OGa2pmVqBm+sgljQHGVCWNjYixdU7dB7i0j2y2lHQXMAM4MiKmNFyBGv0tmvVj0s3K54Apkq7N+zsC4xvMf6dWK2ZmNpSaGbWSg3a9wP2mvOT37sAxdQ5PAt4aES9J2gX4PbB+4zWYV38t8srIlCnA1VXptzeaeeXrhKRVgMWr0h/v8yIzsw4oYNTKzsCkiHhyvrIiXqx6fY2ksyWtFBHPtFJQf4tmnddKhtUk7Q78EFidNEHorcC9pJEsZmZdo6e37c+i/wx9dKtIWg14MiIiP5B+BPBsqwU1MmplXeAU0tDB6lb1Bg3k/11gC+DPefXD7YH9WqyrmVlh2jkhKC9JsgPwpaq0Q1I5cQ7paWmHSpoDvArsE9F6DRq52XkBcDLwv6SvCgeSp+s3YHZEPCtphKQREfFXSae1VlUzs+L0tnHUSl79dcWatHOqXp9JmlvTFo18l1gyIq7NhT8UEceRAnoj/iNpaeBG4GJJpwMvD3CNmdmQa9eEoE5opEX+el4066H81WA6MLK/CyStB6wK7EH62vANYF9SH/nhg6qxmVkBhvVaK6QgvBTwVVJf+bLAQQNccxpwTNXDJXqBCyW9GzgV2G2gQpdY/QMDnWJm1jbt7FoZao0smnVHfjmLuQ+XGMiqEXF3nbzuljSq4dqZmQ2RAkatDJn+JgSNo5+bmhHxiX7yXa6fY37Um5l1nRL3rPTbIh/MHdUJkr4YEedWJ0r6AjBxEPmamRViWHatRMQNg8j368A4SfsyN3CPJj0i7uODyNfMrBDdOBqlUY3c7GxanpK6VZ4AtFFOvjoi/lJEeWZmgzXQ8yu7WSGBvCIi/gr8tcgyzMzaIVgAWuSSFouI14usjJlZp8wpcdfKgONtJG2e1w9/IO9vIuknhdfMzGwIBWp46zaNDJw8A9iVvDJXRNwFbF9kpczMhlpvE1u3aaRrZUREPCbN8ynUU1B9zMw6ohtb2o1qJJA/kdfLjfwg0cOB+4utlpnZ0OrGlnajGgnkh5K6V9YGngT+nNPMzIaNnuHcIo+Ip0gPEDUzG7ba/6S3odPIE4LOpc4yBBExps7pZmal1DucW+SkrpSKxUlT7J8opjpmZp0xXBfNAiAiLqvel/RL4ObCamRm1gHD/WZnrbeRnv5jZjZs9GoYd61Iep653zpGAM8BRxdZKTOzoVbmyTH9BnKlWUCbkJ7TCdAbUeYn25mZ1VfmUSv9TtHPQfuaiOjJm4O4mQ1Lvajhrds0stbKZEnvKbwmZmYdFE1s3aa/Z3YuHBFzgPcA4yU9BLwMiNRYf+8Q1dHMrHBl7lrpr4/8TuC9wO5DVBczs44ZrsMPBRARDw1RXczMOqZnmLbIV5Z0RF8HI+JHBdTHzKwjhmuLfCFgaejCW7RmZm02XAP5zIg4achqYmbWQSV+ZOfAfeRmZguCdrbIJT0KzCJNGJ0TEaNrjgs4HdgFeAX4fERMarW8/gL5h1vN1MysbAqYor99RDzTx7GdgfXz9n7gp/n/LekzkEfEc61mamZWNkM8jnwP4KI8W/52SctJektEzGwls0ZmdpqZDXu9TWySxkiaULXVPmgngOskTaxzDGAN5n2uw7Sc1pJWlrE1Mxt2mukjj4ixwNh+TtkmIqZLWgW4XtJ9EXHj4GrYN7fIzcxo71orETE9//8pYBywec0p04G1qvbXZO4qs01zIDczI/WRN7r1R9JSkkZWXgM7AvfUnHYl8DklWwAvtNo/Du5aMTMD2jpqZVVgXBphyMLAJRHxJ0mHAETEOcA1pKGHD5KGHx44mAIdyM3MgN42LVAbEQ+THshTm35O1esADmtLgTiQm5kBw3eKvpnZAqMbHxjRKAdyMzPcIjczK705Km+b3IHczAx3rZiZlZ67VszMSq5dww87wYHczAx3rZiZlZ67VszMSq6nxG1yB3IzM9wiNzMrvXCL3Mys3MrcIvd65F3uoztux5R7buS+qTfzraPatlialZzfF+3XSzS8dRsH8i42YsQIzjj9FHbdbT/evcn2fPrTe/KOd6zf6WpZh/l9UYx2PiFoqDmQd7HN3/ceHnroUR555HFmz57N5Zdfwe67fbTT1bIO8/uiGHOIhrdu40DexVZfYzWemDbjzf1p02ey+uqrdbBG1g38vihGNPFftyk0kEv6hKQHJL0g6UVJsyS92M/5YyRNkDSht/flIqtmZjaP3ia2blP0qJUfALtFxL2NnBwRY4GxAAsvukb3fewNsRnT/81aa67+5v6aa7yFGTP+3cEaWTfw+6IY3djSblTRXStPNhrEbX7jJ0xmvfXexqhRa7HIIouw99578Ierrut0tazD/L4ohlvkfZsg6TLg98DrlcSI+L+Cyx0Wenp6+NrXj+Oaqy9hoREjuODCy5g69f5OV8s6zO+LYvREeVvkigIrL+kXdZIjIg4a6Fp3rZhZo+a8MV2DzeOzb/14wzHnksfGDbq8diq0RR4RBxaZv5lZu7iPvA+SNpB0g6R78v7Gko4rskwzs1aUuY+86Jud5wLHALMBIuKfwD4Fl2lm1rQyT9Ev+mbnkhFxpzRPd9Kcgss0M2tambtWig7kz0hal7w8gaRPATMLLtPMrGllHrVSdCA/jDTB5+2SpgOPAPsVXKaZWdO6scukUUWPWnkY+IikpYARETGryPLMzFrVjTcxG1VoIJd0RM0+wAvAxIiYXGTZZmbNKHMfedGjVkYDhwBr5O1LwE7AuZK+VXDZZmYNa9eoFUlrSfqrpKmSpkj6Wp1ztsuLCU7O2/GDqXvRfeRrAu+NiJcAJH0HuBrYFphIWlTLzKzj2jjLfQ7wzYiYJGkkMFHS9RExtea8myJi13YUWHQgX4WqNVZI48lXjYhXJb3exzVmZkOup01dKxExkzw6LyJmSbqX1CNRG8jbpuhAfjFwh6QrAAG7Apfkm5+F/VBmZs1qZtSKpDHAmKqksXkZ7trzRgHvAe6ok82Wku4CZgBHRsSUZuo7TzlFLpoFIGk0sHXevSUiJjRynRfNMrNGtWPRrA+vuWPDMeeGadcNWJ6kpYG/A6fUrvgqaRmgNyJekrQLcHpEtPzg1cIf9RYREyLidODVRoO4mdlQa+cUfUmLAL8DLq63bHdEvFi5dxgR1wCLSFqp1boP5TM7DxnCsszMmtKuZ3YqjbM+D7g3In7Uxzmr5fOQtDkpFj/bat2L7iOv1lXr95qZVWvjFP2tgf2BuyVV5st8G1gbICLOAT4FHCppDvAqsE8Mop97KAP5bkNYlplZU9o1RT8ibmaAhmtEnAmc2ZYCKX5m52LAJ4FRwMKVVRAj4qQiyzUza5bXWunbFeQp+cw7ntzMrKsUPYKvSIXP7IyInQouw8xs0MrcIi961Mqtkt5dcBlmZoPWrlErnVB0i3wb4POSHiF1rQiIiNi44HLNzJrSE+VdyLboQL5zwfmbmbVFmfvIC+1aiYjHgLWAD+XXrxRdpplZK/zw5T7kZWtHAxsCvwAWAX7F3LVXzMy6Qjf2fTeq6K6Vj5NW/poEEBEz8vq8ZmZdpbfEXStFB/I3IiIkBUBevtbMrOu4Rd63yyX9DFhO0heBg4BzCy7TzKxpHrXSh4j4X0k7AC+S+smPj4jriyzTzKwV7lrpRw7cDt5m1tXK3LVSyFBASQdLOqpqf5qkFyXNkuR1yc2s6/RGNLx1m6LGdB8CnF+1/3RELAOsDHymoDLNzFrmKfrzU0RUP+3iNwAR8ZqkJQoq08ysZT3R0+kqtKyoQL5c9U5EnAogaQTQ8nPpzMyK4in687tO0sl10k8CriuoTDOzlnmK/vyOAn4u6UHgrpy2KTAe+EJBZZqZtazMLfJCAnlEvAx8RtI6wLty8tSIeKiI8szMBqsbR6M0qujVDx8mLZS1PfBlSXsWWZ6ZWas8aqUPks4G1gMuzUmHSNohIg4rslwzs2Z5in7fPgS8I3Lnk6QLgSkFl2lm1jT3kfftQWBt4LG8v1ZOMzPrKmXuIy86kI8E7pV0JxDA5sAESVcCRMTuBZdvZtYQt8j7dnzB+ZuZtUU3jg9vVNHL2P69yPzNzNqlzC3yQocfStpC0nhJL0l6Q1KPpBeLLNPMrBU90dvw1m2K7lo5E9iHtGjWaOBzwAYFl2lm1rQy3+wstEUOEBEPAgtFRE9E/ALYqegyzcyaFRENb92m6ED+iqRFgcmSfiDpG0NQpplZ09o5s1PSTpL+JelBSUfXOb6YpMvy8TskjRpM3YsOqvvnMr4CvEwaR/7Jgss0M2tau1rkkhYCzgJ2Bt5JWnfqnTWnHQw8HxHrAT8Gvj+YuhfSRy5pZWDliJiak14DTpT0LuCFIso0MxuMNvaRbw48mNeaQtKvgT2AqVXn7AGckF//FjhTkqLFfpuibnb+BDi7TvoKwLHAZwfKYM4b09XuSpWVpDERMbbT9bDu4vdFezUTcySNAcZUJY2t+lusATxRdWwa8P6aLN48JyLmSHoBWBF4ptl6Q3FdK+tFxI21iRFxE7BxQWUOZ2MGPsUWQH5fdEhEjI2I0VVbRz9QiwrkI/s5tkhBZZqZdYPppPuBFWvmtLrnSFoYWBZ4lhYVFcgflLRLbaKknYGHCyrTzKwbjAfWl/S2PGpvH+DKmnOuBA7Irz8F/KXV/nEoro/868DVkvYGJua00cCWwK4FlTmcuR/U6vH7ogvlPu+vANcCCwHnR8QUSScBEyLiSuA84Jf5cZjPkYJ9y1TU4HZJi5Fuam6Uk6YAl0TEa4UUaGa2gCoskM9XkLRrRFw1JIWZmS1AhjKQT4qI9w5JYWZmC5ChnC4/LMaFS1pN0q8lPSRpoqRrJDW9EJikhSWdKukBSZPzdmwRdbbmSApJP6zaP1LSCVX7YyTdl7c7JW2T08flv+ODkl6o+rtuVaeMDfJ75wFJkyRdLmnVFuu7tKSf5vfkpPy+/GIreVk5FRLIJW1RJ/lLRZQ1lCQJGAf8LSLWjYjNgGOAVv4BngysDrw7IjYFPoCHZnaL14FPSFqp9oCkXUnv5W0i4u3AIcAlklaLiI/nv+UXgJsiYtO83VqTx+LA1cBPI2L9/E31bGDlFuv7c+B5oJLXTqTJd7agaGZ9gSbWIZhURL6d3kgPk76xTvrSwA3AJOBuYI+cPgq4D7gYuJc0FXfJvD0LjOynrP2AO4HJwM9IK0gCvAScAtwF3A6s2sf1BwP35zzOBc7M6SsDvyMNkRoPbJ3TTwDOB/5GGiL61U7/vjv4d36J9AF9St4/Ejghv74J+FDN+d8Fvlu1vx1wVT/5HwRc1MexUbmMSXnbqirPG0kfAP8CziE1xNbNf68R/ZR3VP5b/xM4saqce/N7YwpwHbBEH9f/dy7zZuBS4Micvi7wJ9LItJuAt+f0C4AzgFtz3T7V6b/pcN+8EmFzNmLucMpqrwEfj9Qa2h74YW69A2wInB0R7wBeBL4MrAc8HhGz6hUi6R3Ap0lBdlOgB9g3H14KuD0iNiH9w57vK7Sk1Un/+LYAtgbeXnX4dODHEfE+0gJmP6869nbgo6S1Ir4jaUH+hnAWsK+kZWvS38X874EJOb1Rfb2PAJ4CdsjvpU+TAmLF5sDhpIWY1gU+kcu9K6L+0w4k7Qisn6/dFNhM0rb58PrAWRHxLuA/1FnQTlLlfbIJaRGo0VWHxwKHR/pmeiTzLsvxFmAb0nDj7/Xxs1qbFDWOfJ3KA5brieH30GUBp+Z/IL2kdRQq3S1PRMQt+fWvgK+SWj9zL5YOBL5GWmthK+DDwGbA+Px5sATpHzjAG0Bl9M9EYIc69dkc+HtEPJfz/w1zH+jxEeBcOnMCAAAHoUlEQVSdcz9nWEbS0vn11RHxOvC6pKfyzzCt8V/D8BERL0q6iPT3enUIi16EtIBS5QO8+v7LnTF3IaZLSYHyL9UX5/ssewGrRMTqwI55+0c+ZWlSAH8ceCQiJuf0iaRWeq2tgSsiDRt+TdIfcjlLk96rv6l6Ly1Wdd3v84fL1Fb7/q1xRQXyp4EfDnhW+UwhzcKqtS+py2KziJgt6VFg8XysdlhQAA8Ca0saGRGzIj1w4xeS7iFNIBBwYUQcU6es2ZG/v5L+oS+cl82stPCuJH0l78sIYIuoGc+f/zG+XpXUQ/FPkOp2p5F+l7+oSptK+pCtDqCbkd4bdUl6P6l7DNIDyacAH+zj9G8AT5JawCNI3/Yq6r2XpgKbSBoREb0RcQpwiqSXKsUD/xMRP6u+MK9/Xfv3XkLSWsAfcto5ff1MuW7/yd8Y66nOe1gMdOhmRXWtzIqIv/e1FVTmUPgLsFhe+QwASRsDbwWeykF8+7xfsbakLfPrzwI3R8QrpJldZ+YbX5U1jBfN590AfErSKvnYCpKq85xHpKcvVW6sHU/qD/2gpOXzOg7VX5mvI309r9S/r3+IC7z8jeZy0v2Gih8A35e0Irz5+/s89Vf7rORzR9Xf50rgEmArSR+rnCNpW0kbkdbcmJlbs/uTPtgrNs/TvkeQul1ujvQErgnAyfk9VLmZWgme1wIHVb51SVqj8r7qo65PVNX1HOAWYDdJi+c8ds3nvQg8ImmvnK8kbdLnL9MKVVQgf7SgfDsqt4Q/DnwkD/WaAvwPcA0wWtLdpOeS3ld12b+AwyTdCywP/DSnHwvMBO6R9A/SzaILgRmR1nE/DrhO0j+B60l9jo3WczpwKulG5y2kv0dlHfiv5rr+U9JU0qgL69sPgTdHr+RAfD5wq6T7SDcL94uImY1mGBGvkgLi4Xn44VTSvZOnSR8IB0i6i3TP4uWqS8eTnoN7L/AIaQQVpFEyK5LWOJpAer98K5d1HemD47b8/vwt/S9qV1vX8aRvef8E/ki6mV95L+0LHJzrOoW0xrZ1QJFT9FcBDmPuTaAppJt+TxZSYBfKX1+vioiNBji1iLKXjoiXcot8HGm9h3EDXWfdSdJ2pNEiQ75WUdV7aUnSDfYxEdFf950NsaLGkW9Naj0AXJQ3gDvyMSveCZImA/eQWm+/73B9rLzG5vfSJOB3DuLdp5AWuaTbgUMj4h816ZsCP4uI2qdlmJlZi4rqI1+mNogD5KFODffPmZnZwIoK5JK0fJ3EFQos08xsgVRUUP0xacTFByWNzNt2pLvePy6oTDOzBVIhgTzSg0hPJK1B8WjeTgJOrp2YYN1HUk9ete8eSb/JoxVazWs7SVfl17tLOrqfc5eT9OUWyjhB0pGNptecc4GkepO8+jp/VJ64ZdY1CuvmiIirImLbiFgxb9tGxB8GvtK6wKt5QshGpCUB5hlrnid/NP3eiYgrI6K/dTeWI42nNrMmFDIFW9Lx/RyOiPhuEeVaIW4CNs5j4q8F7iBNSd9F0oakb16LAQ8BB+bxxjuRpre/QloxDwBJnwdGR8RX8vob5wDr5MOHkiYrrZuHul0fEUdJOgrYO5cxLiK+k/M6lvTw2qeAJ+h7EapK2V8ExpBmzz4I7J9n2EKa4HU0sAxwRERclWdJfo+06uBipMWlaqe5v4s0fX9RUqPokxHxwIC/UbM2K6pF/nKdDdJU5/8qqExrszyZaGfSbD5Iiy2dnVfLe5k0+/QjeaW+CcAReXr4ucBupIC/Wh/Zn0Fa2GsT4L2kCWNHAw/lbwNHqY+V+yRtRnpY7abALsD7Gvhx/i8i3pfLu5d5p92PymV8DDgn/wwHAy/kVSLfB3xR0ttq8jwEOD2vNzKaBXSBMeu8QlrkEVH9dJWRpJX9DgR+zfBcTGu4WSK3iiG1yM8jPQTjsYi4PadvQVpO9Za84NaiwG2kaeWPVFqmkn5FagnX+hBpOQMiogd4oc5Ip75W7htJap2/ksvoc6XNKhtJOpnUfbM06dtFxeV5bZMHJD2cf4YdSd9EKv3ny+ay76+67jbgWElrkj4o3Bq3jihsdbs81PAI0noMFwLvjYjniyrP2urV2lXtcrCuXvdDpO6Pz9Sc185FuPpaue/rLeR1AbBnRNyVu3i2qzpWb1VBkdbarg74lWUX0kkRl0i6g9SSv0bSlyJinmVlzYZCUVP0/x9piv4s0qPMTnAQH3ZuB7aWtB6ApKWUnl16HzBK0rr5vM/0cf0NpH5xJC2k9ACHWcw7YayvlftuBPaUtET+xrdbA/UdCcxUeljGvjXH9pI0Itd5HdJCZ9cCh+bzK8/YXKr6IknrAA9HxBnAFcDGDdTDrO2KapF/k7Qe8XGkr56VdJFudi5TULk2RCLi6dyyvVRS5YECx0XE/UrL/F4t6RVS10y92bxfI63hcTBpLexDI+I2Sbfk4X1/zP3k7yCt3AfpEWz7RcQkSZeRHnf3FHPX9enPf5Nu1D6d/19dp8dJK0UuAxwSEa9J+jmp73ySUuFPA3vW5Lk3sL+k2cC/SStOmg25wlY/NDOzoeHp8mZmJedAbmZWcg7kZmYl50BuZlZyDuRmZiXnQG5mVnIO5GZmJff/ASh35MgsekxuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"CapGen\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20,random_state=20000)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "\n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['CapGen-gen','NOT-CapGen-gen']); ax.yaxis.set_ticklabels(['CapGen-gen' , 'NOT-CapGen-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "#     auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "#     print('ROC AUC: %f' % auc)\n",
    "    print(\"CapGen label is not good\")\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: 0.634146\n",
      "ROC AUC: 0.384615\n",
      "Accuracy: 0.933333\n",
      "Precision: 1.000000\n",
      "Recall: 0.500000\n",
      "F1 score: 0.666667\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEnCAYAAAC0Z8hNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XecXFXdx/HPN6EnoQQFpYaOonRC9REsFEXABigIKBIpIoqChY5dFOlCEKQj8NCLFHkEAWkhhN6kE3oxVCHJ/p4/zlky2ezs3pnMyc4k3zev+9q5d+6cc3aY/PbMuef+jiICMzPrXIMGugFmZjZ9HMjNzDqcA7mZWYdzIDcz63AO5GZmHc6B3MyswzmQ23STNLekSyVNkHTedJSznaSrW9m2gSDpb5J2HOh22KzDgXwWIunrksZIelPSczngbNCCor8CLAwsGBFfbbaQiDgzIjZuQXumImlDSSHpwh7HV8nHr6tYzsGSzujvvIjYLCJObbK5Zg1zIJ9FSNobOAL4FSnoLgEcB2zZguKXBB6OiEktKKuUl4B1JS1Yc2xH4OFWVaDE/6ZshvOHbhYgaT7gUGCPiLggIt6KiIkRcWlE7JPPmVPSEZKezdsRkubMz20o6RlJP5T0Yu7NfzM/dwhwILBN7unv3LPnKmlE7vnOlvd3kvSYpDckPS5pu5rjN9a8bj1Jt+chm9slrVfz3HWSfi7pplzO1ZI+0Mfb8B5wEbBtfv1gYBvgzB7v1ZGSnpb0uqQ7JH0iH98U+FnN73lXTTt+Kekm4G1g6Xzs2/n5P0k6v6b830q6VpIq/w8064cD+axhXWAu4MI+ztkPWAdYFVgFGAnsX/P8h4D5gEWBnYFjJS0QEQeRevnnRMTQiDipr4ZIGgIcBWwWEcOA9YBxvZw3HLg8n7sgcDhweY8e9deBbwILAXMAP+qrbuA0YIf8eBPgXuDZHufcTnoPhgNnAedJmisiruzxe65S85pvAKOAYcCTPcr7IfDx/EfqE6T3bsdwbgxrIQfyWcOCwMv9DH1sBxwaES9GxEvAIaQA1W1ifn5iRFwBvAms0GR7uoCPSZo7Ip6LiPt6OefzwCMRcXpETIqIs4EHgS/UnPOXiHg4It4BziUF4Loi4l/AcEkrkAL6ab2cc0ZEvJLr/AMwJ/3/nqdExH35NRN7lPc26X08HDgD2DMinumnPLOGOJDPGl4BPtA9tFHHIkzdm3wyH3u/jB5/CN4GhjbakIh4izSksSvwnKTLJa1YoT3dbVq0Zv/5JtpzOvBdYCN6+YYi6UeSHsjDOf8hfQvpa8gG4Om+noyIW4HHAJH+4Ji1lAP5rOFm4F1gqz7OeZZ00bLbEkw77FDVW8A8Nfsfqn0yIq6KiM8CHyb1sk+s0J7uNo1vsk3dTgd2B67IveX35aGPfYGtgQUiYn5gAikAA9QbDulzmETSHqSe/bO5fLOWciCfBUTEBNIFyWMlbSVpHkmzS9pM0u/yaWcD+0v6YL5oeCBpKKAZ44D/kbREvtD60+4nJC0sacs8Vv4uaYimq5cyrgCWz1MmZ5O0DfBR4LIm2wRARDwOfJJ0TaCnYcAk0gyX2SQdCMxb8/wLwIhGZqZIWh74BbA9aYhlX0l9DgGZNcqBfBaRx3v3Jl3AfIk0HPBd0kwOSMFmDHA3cA8wNh9rpq5rgHNyWXcwdfAdlNvxLPAqKaju1ksZrwCbky4WvkLqyW4eES8306YeZd8YEb1927gKuJI0JfFJ4L9MPWzSfbPTK5LG9ldPHso6A/htRNwVEY+QZr6c3j0jyKwV5IvnZmadzT1yM7MO50BuZtbhHMjNzDqcA7mZWYdzIDcz63B93ek3oCa+/Jin09g05l7kEwPdBGtDk94bP91JyBqJObN/YOm2SnrWtoHczGyG6po80C1omgO5mRlA9HaDcWdwIDczA+hyIDcz62gxuZ0XuOqbZ62YmUEaWqm69UPSyXk1rXtrjh0m6UFJd0u6UNL8dV77hKR7JI2TNKZK0x3IzcwgXeysuvXvFGDTHseuAT4WESuTErP9tOeLamwUEatGxJpVKnMgNzODlvbII+KfpOyetceurlmc5RZgsVY13YHczAzSxc6Km6RRksbUbKMarO1bwN/qPBfA1Xnx70rl+mKnmRkQDUw/jIjRwOhm6pG0H2kBkzPrnLJBRIyXtBBwjaQHcw+/LgdyMzOAGTBrRdJOpAVTPh11FoOIiPH554uSLgRGAn0Gcg+tmJlBqy92TkPSpqSVrrbouV5szTlDJA3rfgxsDNzb27m1HMjNzKDV0w/PJi16voKkZyTtDBxDWhf2mjy18Ph87iKSrsgvXRi4UdJdwG3A5RFxZX/1eWjFzAxaemdnRHytl8Mn1Tn3WeBz+fFjwCqN1udAbmYGzrViZtbxnGvFzKyzRdfEgW5C0xzIzczAPXIzs47nMXIzsw7nFYLMzDqce+RmZh2ugxeWcCA3MwNf7DQz63gO5GZmnS3CFzvNzDqbe+RmZh3Os1bMzDqcZ62YmXU4D62YmXU4D62YmXU498jNzDqcA7mZWYfzxU4zsw7nMXIzsw7noRUzsw7nHrmZWYdzj9zMrMM5kJuZdbjJzn5oZtbZ3CM3M+twvthpZtbh3CM3M+twEQPdgqY5kJuZgXvkZmYdz7lWzMw6W3R17tDKoIFugJlZW+jqqr71Q9LJkl6UdG/NseGSrpH0SP65QJ3X7pjPeUTSjlWa7kBuZgZp+mHVrX+nAJv2OPYT4NqIWA64Nu9PRdJw4CBgbWAkcFC9gF/LgdzMDKArqm/9iIh/Aq/2OLwlcGp+fCqwVS8v3QS4JiJejYjXgGuY9g/CNDxGbmYGMKn4xc6FI+K5/Ph5YOFezlkUeLpm/5l8rE/ukbeZ/X91OP/z+W3Zavtd3z929OjT+OIOu/HlHfdgl+//jBdfemUAW2jtYJONN+S+e//Jg/ffyL777DHQzZk5RFTeJI2SNKZmG9VYVRFAy66uOpC3ma0+91mOP/wXUx375nZf5sLT/sT5px7LJ9dfmz/95awBap21g0GDBnHUkb9k8y9sz8dX2YhtttmKj3xkuYFuVudr4GJnRIyOiDVrttEVanhB0ocB8s8XezlnPLB4zf5i+VifHMjbzJqrfpz55h021bGhQ4a8//idd/6LNKNbZe1k5Fqr8eijT/D4408xceJEzj33Yrb4wiYD3azO18Ix8jouAbpnoewIXNzLOVcBG0taIF/k3Dgf61PxMXJJiwJL1taVLwRYA4484RQuufJahg0ZwslH/2agm2MDaJFFP8TTzzz7/v4z459j5FqrDWCLZhItTJol6WxgQ+ADkp4hzUT5DXCupJ2BJ4Gt87lrArtGxLcj4lVJPwduz0UdGhE9L5pOo2iPXNJvgZuA/YF98vajPs5/f9zpz6edXbJpHWev7+zEtReezuc33oizzr90oJtjNvNp7ayVr0XEhyNi9ohYLCJOiohXIuLTEbFcRHymO0BHxJiI+HbNa0+OiGXz9pcqTS/dI98KWCEi3q1ych5nGg0w8eXHOvc2q4I233gjdvvRgXz3298Y6KbYAHl2/PMsvtgi7+8vtuiHefbZ5wewRTOHmNS5C0uUHiN/DJi9cB0zvSefnnKt4/9uuJmlllxsAFtjA+32MeNYdtmlGDFicWaffXa23npLLr3s6oFuVudr7Q1BM1TpHvnbwDhJ1wLv98oj4nuF6+1Y+xz0G26/827+85/X+fRW27P7zt/ghptv54mnnkGDxCIfWogD99lzoJtpA2jy5Mns9f39ueLysxg8aBCnnHoO99//8EA3q/N1cK4VRcEcvPXyBETEqb0dr+WhFevN3It8YqCbYG1o0nvjp3su11sHf61yzBly8NltNXesaI88Ik6VNDewREQ8VLIuM7Pp0sE98tKzVr4AjAOuzPurSrqkZJ1mZk3xGHldB5MyeF0HEBHjJC1duE4zs4Z18qyV0oF8YkRM0NS3IrbfnzMzsw4eWikdyO+T9HVgsKTlgO8B/ypcp5lZ4zo4kJeeR74nsBJp6uHZwOvA9wvXaWbWOI+R9y4i3gb2y5uZWfvq4B550UAu6VKmzbk7ARgDnBAR/y1Zv5lZVTGp/XraVc2IW/TfBE7M2+vAG8Dyed/MrD20cPHlGa30xc71ImKtmv1LJd0eEWtJuq9w3WZm1XXw0ErpHvlQSUt07+THQ/Pue4XrNjOrrvzCEsWU7pH/ELhR0qOAgKWA3SUNYcpq0mZmA65k3qnSSs9auSLPH18xH3qo5gLnESXrNjNrSBv2tKsqvmZnRLwbEXcBe3iWipm1q5jUVXlrN8XX7Kyx5gysy8ysMR3cI5+RgfzFGViXmVlj2q+jXVmxoRVJgyX9vns/IjYtVZeZ2fSKrqi8tZtiPfKImCxpg1Llm5m1VBsG6KpKD63cmReSOA94q/tgRFxQuF4zs8Z08NBK6UA+F/AK8KmaYwE4kJtZW4lJ7pH3KiK+WbJ8M7NWacex76qKBHJJ+0bE7yQdzbTZD4mI75Wo18ysaR5amcYD+eeYQuWbmbVUG64XUVmRQB4Rl+afzqdiZp1hZg7kkr4EXBMRb0j6CbA68KuIGFfhtR8Efgx8lHThE4CI+FTdF5mZDYCYNNAtaF6VG4IOzkF8PeBzwJnA8RXLP5M0zLIUcAjwBHB7E+00Myuqg5fsrBTIJ+efm5OWZ7sYmLNi+QtGxEnAxIi4PiK+xdRTEc3M2kInB/IqY+TPSToW2BRYU9IcVL+1f2JNGZ8HngWGN95MM7Oy2jFAV1UlkG9NGlI5OiJek7QI8JOK5f9C0nykBSaOBuYFftBUS83MSgoNdAuaVjeQS5q3ZvfKmmNvAjdVKTwiLssPJwAbNdlGM7PiWtUjl7QCcE7NoaWBAyPiiJpzNgQuBh7Phy6IiEObrbOvHvl9pJt5av9Mde8HsERvL6olaWngSGBd0uSem4EfRMRjzTbYzKyErkmt6ZFHxEPAqpCywALjgQt7OfWGiNi8FXXWDeQRsXgLyj8LOBb4Yt7fFjgbWLsFZZuZtUyUGVr5NPBoRDxZovBulS5aStpW0s/y48UkrVGx/Hki4vSImJS3M6iZT25m1i4KzVrp7rz2Zl1Jd0n6m6SVpqft/QZySceQxre/kQ+9TfV55H+T9BNJIyQtKWlf4ApJwyV59oqZtY3oUuVN0ihJY2q2UT3LyzP8tiCl8e5pLLBkRKxCmghy0fS0vcqslfUiYnVJdwJExKu5gVVsnX9+p8fxbUnj7EtXLMfMrKhoIPlhRIwGRvdz2mbA2Ih4oZfXv17z+ApJx0n6QES8XL0VU1QJ5BMlDSJnMZS0IBWzEkTEUs00ysxsRuua1PKVL79GnWEVSR8CXoiIkDSSNDrySrMVVWn5scD5wAclHQLcCPy2SuGS5pG0v6TReX85SS25Smtm1koR1bf+SBoCfJaaRXQk7Spp17z7FeBeSXcBRwHbRjTynWBq/fbII+I0SXcAn8mHvhoR91Ys/y/AHcB6eX88abzosrqvMDMbANHVulkrEfEWsGCPY8fXPD4GOKZV9VX9LjGYdLv9ew28BmCZiPhdfi0R8TZTz0s3M2sLEaq8tZsqs1b2I43zLAIsBpwl6acVy39P0txMGV9fBni3ybaamRUzsyfN2gFYLfemkfRL4E7g1xVeexDp9v7FJZ0JrA/s1FxTzczK6WrDnnZVlbIf9jhvtnysXxFxjaSxwDqkIZW9mp1eY2ZWUtfkls9amWH6Spr1R9KQyKvAfZKuyvsb08DiEBHxCnB5LnN5Sb+OiF2mq9VmZi3W/JyRgddXj7x7Zsp95ECc3dJfoZJWBn5PGle/iDSF8RhSjpU/NNVSM7OCWjlrZUbrK2nWSdNR7onAn0jZDjcFxgGnAttFxH+no1wzsyJm6jHyPNPkl0y7gPLyfbxszog4JT9+SNJeEbHv9DTUzKykdpxWWFWVi52nAL8gDZVsBnyTPJ2wD3NJWo0pc8bfrd2PiLFNtdbMrJCZdYy82zwRcZWk30fEo8D+ksYAB/TxmueAw2v2n6/ZD7wAs5m1mcldM+GslRrv5qRZj+Y8AeOBYX29ICK8rJuZdZSZvUf+A2AI8D3SWPl8wLeqFC5pLmB3YANST/wG4PgqFzyXWX7LKlXYLGaRoU5jb2XM1Bc7I+LW/PANpiwuUdVp+XVH5/2vA6cDX22wHDOzombKi52SLqSPi5oR8aUK5X8sIj5as/8PSfc30D4zsxliZu2RtyLF4lhJ60TELQCS1gbGtKBcM7OWmjwzBvKIuLYF5a8B/EvSU3l/CdK88ntSFbFyC+owM5tuM+XQSotsWrh8M7OWaMPstJUVnTgZEU8CiwOfyo/fAgZFxJN538ysLQSqvLWbyj1ySXNGREOLQkg6CFgTWIG07NscwBmkvORmZm2jq4PnkVdZIWhkHtN+JO+vIunofl7W7YvAFqSeOBHxLP3cTGRmNhC6UOWt3VQZWjkK2Bx4BSAi7gKq3rn5Xl4ZunuptyHNNNLMrLTJqPLWbqoE8kG9jGdPrlj+uZJOAOaXtAvwd1KKWzOztjKzj5E/LWkkEJIGA3sCD1cpPCJ+L+mzwOukcfIDI+KapltrZlZIJ89aqRLIdyMNrywBvEDqVe/WQB0Pk+aM/13SPJKGRcQbjTfVzKycmTqQR8SLwLbNFJ6HU0YBw4FlgEWB44FPN1OemVkp7ThkUlWVFYJOpJecKxExqkL5ewAjgVvzax6RtFCjjTQzK62Dl+ysNLTy95rHc5GmFD5dsfx3I+I9Kb1Dkmaj/9WFzMxmuHacjVJVlaGVc2r3JZ0O3Fix/Osl/QyYO1/03B24tOFWmpkV1slj5M3cor8UsHDFc38CvATcA3wHuALYv4k6zcyK6pIqb+2myhj5a0wZDhkEvEoK0P2KiC5JFwEXRcRLTbfSzKywTh7z7TOQKw1ur0JapxOgK9+p2af8uoOA75J7/ZImA0dHxKHT1WIzswJm2qGVHLSviIjJeav6R+sHpMRYa0XE8IgYDqwNrC/pB9PXZDOz1pskVd7aTZUx8nGSVmuw3G8AX4uIx7sPRMRjwPbADg2WZWZWXDSwtZu+1uycLSImAasBt0t6lJTFUKTO+up9lDt7RLzc82BEvCRp9ulttJlZq7VyHrmkJ0gLz08GJkXEmj2eF3Ak8DngbWCniBjbbH19jZHfBqxOSkPbqPeafM7MbEAUGCPfqLcObbYZsFze1gb+lH82pa9ALoCIeLSJcleR9HqdMudqojwzs6Jm8JDJlsBp+brjLZLml/ThiHiumcL6CuQflLR3vScj4vA+nhvcTGPMzAZKi2/RD+BqSQGcEBGjezy/KFPfIf9MPtbyQD4YGAodfN+qmVlFkxo4V9IoUkLAbqN7BOsNImJ8zi11jaQHI+KfLWloL/oK5M95zreZzSqigS5rDto9e9m1z4/PP1+UdCEpeWBtIB9PWpi+22JMuV+nYX1NP3RP3MxmGV0NbH2RNETSsO7HwMbAvT1OuwTYQck6wIRmx8eh7x65c4ab2SyjhbNWFgYuzFlfZwPOiogrJe0KEBHHk/JOfQ74N2n64Tenp8K6gTwiXp2egs3MOkmrZq3kmx9X6eX48TWPg7ReQ0tUyUduZjbTm9kXljAzm+k1Mmul3TiQm5nRnjlUqnIgNzPDQytmZh2vk/ORO5CbmeGhFTOzjjepg0O5A7mZGe6Rm5l1PI+Rm5l1OM9aMTPrcF0dPLjiQG5mhsfIzcw6nmetmJl1uM4N4w7kZmaAZ62YmXU8X+w0M+twnRvGHcjNzAAPrZiZdbzJHdwnHzTQDbD6Djv6UMY+dB3X3HTBQDfF2og/F2V0EZW3duNA3sbOO+tidvjqbgPdDGsz/lyUEQ1s7caBvI3ddvMd/Oe1CQPdDGsz/lyU0ck98uJj5JIGAwvX1hURT5Wu18ysEb7YWYekPYGDgBeY8j4FsHLJes3MGtXJFztL98j3AlaIiFeqnCxpFDAKYIF5FmHonMNLts3M7H3RwYG89Bj500DlwbyIGB0Ra0bEmg7iZjYjdTWwtZvSgfwx4DpJP5W0d/dWuM6ZxtEn/paLrjqDpZcdwa33/p1ttv/iQDfJ2oA/F2V0RVTe2k3poZWn8jZH3qwBe+7y44FugrUhfy7KaL/wXF3RQB4RhwBImici3i5Zl5nZ9GjHaYVVFR1akbSupPuBB/P+KpKOK1mnmVkzJhOVt3ZTeoz8CGAT4BWAiLgL+J/CdZqZNcw3BPUhIp6WplqeenLpOs3MGtXJ0w9LB/KnJa0HhKTZSfPKHyhcp5lZw9pxWmFVpYdWdgX2ABYFxgOr5n0zs7YSEZW3vkhaXNI/JN0v6T5Je/VyzoaSJkgal7cDp6ftpWetvAxsV7IOM7NWmNS6oZVJwA8jYqykYcAdkq6JiPt7nHdDRGzeigpL51o5qpfDE4AxEXFxybrNzBrRqjHyiHgOeC4/fkPSA6RRiZ6BvGVKD63MRRpOeSRvKwOLATtLOqJw3WZmlTUya0XSKEljarZRvZUpaQSwGnBrL0+vK+kuSX+TtNL0tL30xc6VgfUjYjKApD8BNwAbAPcUrtvMrLL+xr57nDsaGN3XOZKGAucD34+I13s8PRZYMiLelPQ54CJgucZaPEXpHvkCwNCa/SHA8BzY3y1ct5lZZa1MmpVn6Z0PnBkR06zJFxGvR8Sb+fEVwOySPtBs20v3yH8HjJN0HSDSzUC/kjQE+Hvhus3MKmvVGLnSjTMnAQ9ExOF1zvkQ8EJEhKSRpE51pXTfvSk9a+UkSVcAI/Ohn0XEs/nxPiXrNjNrxORo2Uzy9YFvAPdIGpeP/QxYAiAijge+AuwmaRLwDrBtNDK208OMuLPzOeBiSQd7poqZtatW3XofETeSRiD6OucY4JiWVMiMXXx5ixlYl5lZQ6KB/9pN8R55jT7/QpmZDaR2XDCiqmI9ckmDJf2g5tAapeoyM5te0cDWbooF8jzF8Gs1+52ck8bMZnJOY1vfTZKOAc4B3uo+GBFjC9drZtaQFs5ameFKB/JV889Da44F8KnC9ZqZNaQde9pVlZ5HvlHJ8s3MWqUdZ6NUVSSQS9o+Is6QtHdvz9e728nMbKBMx/04A65Uj3xI/jmsUPlmZi3loZUeIuKE/POQEuWbmbWaL3bWIWkuYGdgJVJucgAi4lsl6zUza1Qnj5GXvkX/dOBDwCbA9aRFJd4oXKeZWcO6Iipv7aZ0IF82Ig4A3oqIU4HPA2sXrtPMrGHOtVLfxPzzP5I+BjwPLFS4TjOzhrVjT7uq0oF8tKQFgAOAS0irBR1QuE4zs4a1Y0+7qtI3BP05P7weWLpkXWZm06OTZ60UHSOXtKCkoyWNlXSHpCMkLViyTjOzZvhiZ31/BV4Evkxa2uhlUgItM7O24oud9X04In5es/8LSdsUrtPMrGGdnGm7dI/8aknbShqUt62BqwrXaWbWsE7OR66SiWIkvUHKu9JFSl87mCl5ySMi5q332iWGf7z93i0za0tPvXrPdC8l2UjMaUV9rVR61oqTZplZR/CslTqUbC/pgLy/uKSRJes0M2uGZ63UdxywLvD1vP8mcGzhOs3MGuZZK/WtHRGrS7oTICJekzRH4TrNzBrmhSXqmyhpMOlCJ5I+SLrwaWbWVtpxNkpVpQP5UcCFwEKSfkm6KWj/wnWamTVsclfn9jFLz1o5U9IdwKcBAVtFxAMl6zQza4aHVvoQEQ8CDwJIml/SfhHxy9L1mpk1opOHVorMWsnTDEdLukzStyUNkfQH4GGcj9zM2lBEVN7aTake+Wmk1LXnA5sCY4BxwMoR8XyhOs3MmtaO88OrKhXIh0fEwfnxVZK+CmwXnZyVxsxmau04P7yqYmPkeWWg7nwErwDzSRJARLxaql4zs2Z41sq05gPuYEogBxibfwZeLcjM2kwre+SSNgWOJCUK/HNE/KbH83OShqDXIHV0t4mIJ5qtr0ggj4gRJco1MyulVRcx802QxwKfBZ4Bbpd0SUTcX3PazsBrEbGspG2B3wJNr9VQfPqhpC8BG5B64jdExEWl6zQza1QLZ6OMBP4dEY8BSPorsCVQG8i3BA7Oj/8XOEaSoslGFA3kko4DlgXOzod2lfTZiNijv9e2W77fgSRpVESMHuh2WHvx56K1Jr43vnLMkTQKGFVzaHTN/4tFgadrnnsGWLtHEe+fExGTJE0AFiQth9mw0j3yTwEf6f4rI+lU4L7Cdc6MRgH+B2s9+XMxQHLQbpv3vnQa238DS9TsL56PmZnNrMaTYl23xfKxXs+RNBtpgsgrzVZYOpAPAx6QdJ2kf5DGiOaVdImkSwrXbWY2EG4HlpO0VE7bvS3QM95dAuyYH38F+L9mx8eh/NDKgYXLn1W0zVc4ayv+XLShPOb9XdJC84OBkyPiPkmHAmMi4hLgJOB0Sf8GXiUF+6YVXXwZQNKSwHIR8XdJcwOzRcQbRSs1M5uFlF6zcxfS1JoT8qHFAE8/NDNrodJj5HsA6wOvA0TEIzj7oZlZS5UO5O9GxHvdO/nqbFtmppE0WdI4SfdJukvSDyW1/P3JF34fynXcLmnVAnVsIeknrS6300mKnE65e/9Hkg6u2R8l6cG83SZpg3z8wvzZ+LekCfnxOEnr9Sh/hKR3JN0p6YFcxk6FfpcnJN0j6W5J1+chzFbXsaukHVpdrrVe6Yud10v6GTC3pM8CuwOXFq6zWe9ExKoAkhYCzgLmBQ4qUNd2ETFG0jeBw0i38rZMvpgyzawgSbNFxKRW1tVh3gW+JOnXETHVjReSNge+A2wQES9LWh24SNLIiPhiPmdD4EcRsXkfdTwaEavl85cGLsh37P2lwO+zUW7rIaQlFHdpZeERcXxvx/05aj+le+Q/AV4C7iH9I7mCDlizMyJeJN1s8V0lgyUdlnvQd0v6Tve5kvapOX5IPjYi9+rOzD2z/5U0Ty9V3Uy6w6u7rI0l3SxprKTzJA3Nx5+Q9LvcA7tN0rL5+Bck3Zp7gH+XtHA+vpOkY/LjUyQdL+lW4Hc9GyDpc7mtd0g6StJl+fgQSSfn+u6UtGVN2RdIulLSI5KmKbONTSLN9PhBL8/9GNinO8BHxFjgVNLwYFPyLdp7A9+DPt9znFK0AAAKoElEQVTTXj9fkjaU9E9Jl+dvccfX+ZbY83O0fa5jnKQTlHJ/IOlNSX9U+tZ5rdJi6EjaJdd9l6Tzuz+rkg6W9KP8+DpJR0gaA+zVswGSdpb0cK73xJrP3wdzmbfnbf2ask/O5T4m6XvNvs9WOJDn/ONnAPtHxFci4sTpmSs5I+V/hINJY/o7AxMiYi1gLWAXpTmiGwPLkXIrrAqsIel/chErAMdFxEdI1wh276WaTckXfyV9gPRH7jMRsTppMY69a86dEBEfB44BjsjHbgTWyT3AvwL71vl1FgPWi4ja8pA0F+lC9GYRsQbwwZqn9yPNbR0JbAQcJmlIfm5VUoKfjwPbSKq9+aHdHQtsJ2m+HsdXImXsrDUmH58eY4EV8+N672mvn6/8mpHAnsBHgWWAL/VSR+3n6COk/zfr52+Yk4Ht8nlDSNPfViIt/NL9bfOCiFgrIlYBHsjt6c0cEbFmRPyh9qCkRYADgHVI18RWrHn6SOCP+Xf7MvDnmudWBDbJv+NBkmavU6/1o3SulS1IQwdzAEspjQcfGhFblKy3gI2BlSV9Je/PRwrgG+ftznx8aD7+FPB0RNyUj59B6pX9Pu+fqXSjwFBSUIT0j+CjwE1KadvnIPW0up1d8/OP+fFiwDmSPpzPf7xO+8+LiMm9HF8ReCwiul93NlPyR2wMbNHdIwPmYspdutdGxAQASfcDSzJ1bom2FRGvSzqN9P/jnRlQZW3+jnrvab3P13vAbTXJl84mJaD733zePyQNB94kBVJIC52vQcq4BzA38GJ+rgs4Jz8+A7ggP/6YpF8A85M+k1fV+V3OqXN8JHB99zoDks4Dls/PfQb4aG4LpBsCh+bHl0fEu8C7kl4EFiblJbEGlR4jP4j0P/k6gIgYV9PTaGtK45uTSf8IBOwZEVf1OGcT4NcRcUKP4yOY9qJu7f52pN7fYcDRpF6WgGsi4mt1mhS9PD4aODwiLlEavz24zmvfqmnbVaR/MGNIvft6BHw5Ih6a6qC0NmmsudtkZkAWzRY7gtRTrh23vp8UAP+v5tga9JEbKL8X3f/vDwTu7uW01Ui9XKj/ntb7fG1I35+jjYD/AGcCh5C+wQk4NSJ+Wq/dvZR1CrBVRNyldHF2wzrnv5XbNZgp314uYcpaA70ZRPrW+N/agzmwd/rnqG2UHiOf2N1zq9H2Qyt57PB44Jg8FHQVsFv3Vz9Jy+evxFcB39KUsexFlS6UAiwhad38+OukYZD35XIPANaRtCJwC7C+pox/D5G0fM1Ltqn52d1Tn48pORx2pIKI2CQiVo2IbwMPAUvnPzy1dZB/tz1zkEHSalXK7wS553guUw8h/A74raQFAfK3x52A4/oo59b8Xq6aLzBPJb+vvyf9wYX672m9zxfAyDyMN4j0/6fn52gS8H1gh9w7vxb4SvfnUNJwTZnRMoh0OzhM/ZkcBjyX69+OfkTE5Jrf+0DSLemflLSA0sy0L9ecfjVpaKj7PWn5LC0r/xfwPklfBwZLWo70dfZfhets1tySxgGzky6KnQ4cnp/7MzACGJv/Eb5E6sFcncckb87/Nt8Etif1Lh4C9pB0Mqm396eeFUbEO0rT4faJiJ1zb+hspdVDII2ZP5wfLyDpblIvprvXfjBwnqTXSD3Jhr7t5Pp3B66U9BbpH2S3n5N6rnfnIPI40NdsjU7zB+C73Tv5W82iwL8kBfAGsH1EPNdguctIupM0bPIGcFREnJKfq/ee9vr5yq+5nfTNaVngH8CFPSuMiOfysMseEfFzSfsDV+c6JpIu2D5J6lGPzM+/yJQ/3AcAt+Z6byUF9soiYrykXwG3kW43fxDo7sB9Dzg2f3ZnA/4J7NpI+da/orfoK1393o80BihSz+PnPb9mzWxyT+yyiPhYi8p7Aliz55S5FpU9NCLezAHkWOCRiPhjf6+z8lRtumMj5b0ZEUP7P7Opsrs/R7OR/ticHBHT/NGxMor2yCPibVIg369kPTZddpG0I+li6Z1MGfM1a8TBkj5D+iZyNU7FMUMV65Hn4LAXaRoepAs+R0XEaUUqNDObRRXpkecg/n3SVfSxpGGV1UnzZiMiTi9Rr5nZrKhIj1zSLcC2EfFEj+MjgL9GxDotr9TMbBZVavrhvD2DOEA+Nm+hOs3MZkmlAnlfd8zNiLvpbDpoSibIe5VyvvSWJ6ZqWRtqSv6WPrMySpo/T4dstI73c4JUOd7jnFNq7qisUtcISfc22kazkkrNWvlInjfak4ClC9VprVObCfJM0rzf7jn13XciKufSqaxeVsYa85Ny0tS9CcfMplWqR/4R4Au9bJuT8olY57gBWDb3RB9SylNyL7C46mdr3FQpo+JYapI8aeqsjAsr5fm+K2/rAb8h3VAzTtJh+bxpskvm4/spZdu7kSkzo+pSnQx/2WckjcnlbZ7Pr5vxsqbMlTQly+Dd+aY3sxmuSI88Ip7seUzS5hFxWYn6rIx8c8dmwJX50HLAjhFxi6bO1viWpB8DeyultT0R+BTwb+onWjqKlGjpi0q5O4aS0h5/rObbQG12SQGXKGWXfIu0WO2qpM/wWKbNXNjTBRFxYi73F6Tb87tvnR+R61iGlIhqWWAHckbCfKftTZKuZuoUE7sCR0ZEdxK0wf20wayIGZmk5lDAgbwzdKcrgNQjPwlYBHgyIm7Jx+tla1wReDwv64ekM5iSUbHWp0jBkpyZcYKkBXqcUy+75DDgwnzDGZL6Gq7p1leGv3PzMNEjkh7Lv0O9jIQP17zuZmA/SYuR/lA8UqEdZi03IwO5+j/F2sT7Y+TdcrB+q/YQvWRrVGuTIones0t+v4myTqF+hr/eMgzWy0g44v2TIs5SWrDj88AVkr4TEbXZE81miCJj5JJ6myc+zRijdbR62RofBEZIWiafVy8t77XAbvm1g5UWeniDqRM21csu+U9gK0lzSxpGuv7Sn74y/H1V0qDc5qVJCc/6ykhIPrY0KZ/7UcDFwMoV2mHWcqV65MeR7uR8X0TcVqguGwAR8ZJ6ydYYEQ9LGgVcLult0tBMb9n09gJGS9qZlC1yt4i4WdJNeXrf3yJiH/WSXTIixko6B7iLlMXv9l7K76mvDH9PkTL3zQvsGhH/ldRXRsJuWwPfkDQReB74VYV2mLVcqTs7x+blyszMrLBSgfw/pK+/verApd7MzNpWqaGVl0iJ+83MrLBSgfyNiLi+UNlmZlaj1J2dTxQq18zMeii5sMRCpLUCV8qH7gOOi4gXilRoZjaLKjWPfH2mTAk7LW8At+bnzMysRUouLLFbRNzZ4/iqwAkRsXbLKzUzm0WVXFjizp4HI2Icvd8cYmZmTSoVyNVLAiQkDS9Yp5nZLKlUUP0jcLWkT0oalrcNgb/l58zMrEVKzlrZHNiXqWetHBYRlxap0MxsFlUskJuZ2YxR5M5OSQf28XRExM9L1GtmNisqNf3wh70cHkJaXmvBiBja8krNzGZRxYdWcuL/vUhB/FzgDxHxYtFKzcxmIcWWestTDfcmrcZyKrB6RLxWqj4zs1lVqTHyw4AvAaOBj0fEmyXqMTOzcmPkXcC7wCSmXthWpIud87a8UjOzWZSnH5qZdTjfLm9m1uEcyM3MOpwDuZlZh3MgNzPrcA7kZmYd7v8B+vDHQgs1KCwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"DeepRepair\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20 , random_state=1)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "\n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['DeepRepair-gen','NOT-DeepRepair-gen']); ax.yaxis.set_ticklabels(['DeepRepair-gen' , 'NOT-DeepRepair-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "    auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "    print('ROC AUC: %f' % auc)\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: 0.285714\n",
      "ROC AUC: 0.500000\n",
      "Accuracy: 0.800000\n",
      "Precision: 0.333333\n",
      "Recall: 0.500000\n",
      "F1 score: 0.400000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEWCAYAAABLzQ1kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xu8pnO9//HX20xymKFQQjHYaCeRU4Vs7Q67kNQuFBXZTQe782HbZaMDHUlKO2OXc3YoRUSlHY2Ewcix8UMKExMZjGJmrffvj+taM/cs63Dd97qve11r1vs5j+sx93W4v9/vmnXPZ33X5/p+v5dsExERzbPSeDcgIiKGlgAdEdFQCdAREQ2VAB0R0VAJ0BERDZUAHRHRUAnQMWaSVpV0gaSFks4ZQzn7S/pZN9s2HiT9VNI7x7sdMfElQE8ikt4maY6kxyTNLwPJLl0o+s3AusDatt/SaSG2z7T9mi60ZzmSdpNkSecNOr51efxXFcs5UtIZo11n+3W2T+2wuRFLJUBPEpI+ChwHHE0RTDcEvgW8oQvFbwTMs72kC2XVZQHwMklrtxx7JzCvWxWokP9T0TX5ME0CktYEPgscYvuHthfZXmz7AtufKK95uqTjJN1XbsdJenp5bjdJ90j6mKQHyt73QeW5zwCHA/uWPfODB/c0Jc0oe6pTy/0DJd0p6VFJd0nav+X47Jb37STpmjJ1co2knVrO/UrS5yRdUZbzM0nrjPDP8CTwI2C/8v1TgH2BMwf9W31d0p8kPSLpWkkvL4+/FvhUy9d5Q0s7jpJ0BfA4sEl57N/K8/8t6Qct5X9J0qWSVPkbGJNWAvTk8DJgFeC8Ea75NPBSYBtga2BH4LCW888B1gQ2AA4GTpD0TNtHUPTKv297mu3vjNQQSasDxwOvsz0d2AmYO8R1awEXlteuDRwLXDioB/w24CDg2cDKwMdHqhs4DXhH+fpfgJuA+wZdcw3Fv8FawPeAcyStYvviQV/n1i3veTswE5gO3D2ovI8BW5U/fF5O8W/3TmeNhaggAXpyWBv4yygpiP2Bz9p+wPYC4DMUgWfA4vL8YtsXAY8BW3TYnn7ghZJWtT3f9s1DXLMHcLvt020vsX0WcBvw+pZrTrY9z/bfgLMpAuuwbP8GWEvSFhSB+rQhrjnD9oNlnccAT2f0r/MU2zeX71k8qLzHKf4djwXOAD5g+55RyosAEqAniweBdQZSDMNYn+V7f3eXx5aWMSjAPw5Ma7chthdRpBbeC8yXdKGk51doz0CbNmjZ/3MH7Tkd+HfgFQzxG4Wkj0u6tUyrPEzxW8NIqROAP4100vZVwJ2AKH6QRFSSAD05XAk8Aew9wjX3UdzsG7AhT/31v6pFwGot+89pPWn7EtuvBtaj6BWfVKE9A226t8M2DTgdeD9wUdm7XapMQXwS2Ad4pu1nAAspAivAcGmJEdMVkg6h6InfV5YfUUkC9CRgeyHFjbwTJO0taTVJT5P0OklfLi87CzhM0rPKm22HU/xK3om5wK6SNixvUP7nwAlJ60p6Q5mLfoIiVdI/RBkXAZuXQwOnStoXeAHwkw7bBIDtu4B/osi5DzYdWEIx4mOqpMOBNVrO3w/MaGekhqTNgc8DB1CkOj4pacRUTMSABOhJosynfpTixt8Cil/L/51iZAMUQWQO8DvgRuC68lgndf0c+H5Z1rUsH1RXKttxH/AQRbB83xBlPAjsSXGT7UGKnueetv/SSZsGlT3b9lC/HVwCXEwx9O5u4O8sn74YmITzoKTrRqunTCmdAXzJ9g22b6cYCXL6wAiZiJEoN5MjIpopPeiIiIZKgI6IaKgE6IiIhkqAjohoqJEmLoyrxX+5M3cv4ym2/Md9xrsJ0UDzFswZ89om7cScp62zSU/WUkkPOiKioRrbg46I6Kn+vvFuwVMkQEdEAPQ1bznzBOiICMAeasWB8ZUAHREB0J8AHRHRTOlBR0Q0VG4SRkQ0VHrQERHN5IziiIhoqNwkjIhoqKQ4IiIaKjcJIyIaKj3oiIiGyk3CiIiGyk3CiIhmspODjohopuSgIyIaqoEpjjxRJSICih501W0Ukr4r6QFJN7UcW0vSzyXdXv79zNHKSYCOiADoW1x9G90pwGsHHTsUuNT2ZsCl5f6IEqAjIqBIcVTdRmH7cuChQYffAJxavj4V2Hu0chKgIyKgrRSHpJmS5rRsMyvUsK7t+eXrPwPrjvaG3CSMiIC2bhLangXM6rQq25bk0a5LgI6IgF6M4rhf0nq250taD3hgtDckxRERAbhvceWtQ+cD7yxfvxP48WhvSA86IgK6OlFF0lnAbsA6ku4BjgC+CJwt6WDgbmCf0cpJgI6IgK6mOGy/dZhTr2ynnAToiAjIVO+IiMZq4FTvBOiICEgPOiKisZZkwf6IiGZKDzoioqGSg46IaKj0oCMiGio96IiIhkoPOiKioTKKIyKioTzq6p89lwAdEQHJQUdENFYCdEREQ+UmYUREQ/X1jXcLniIBOiICkuKIiGisBOiIiIZKDjoiopncn3HQERHNlBRHRERDZRRHRERDpQcdEdFQDQzQK413A2J5hx19LLvusR97H/Depccu+eWvecP+72GrXXbnplvnjWProgmes/66nHbet7lo9tlc+Ovv846Z+413k1YMdvWtRxKgG2bv3V/Nt4/9/HLH/mGTjTju6P9iu21eOE6tiibp61vCF4/4Grvvsg/7vPYg9n/XW9h0843Hu1kTX39/9a1HkuJomO232Yp759+/3LFNZ2w4Tq2JJlpw/4MsuP9BABYtepw75v2Bddd7NnfMu2ucWzbBTcZhdpJ2Ama01mX7tLrrjZgMNnjeerxgqy244dqbxrspE18DR3HUmuKQdDrwVWAXYIdy236E62dKmiNpzv+cdladTYuY8FZbfVW+cfKXOfqwY1j02KLxbs6E5/7+yluv1N2D3h54gV0tq257FjALYPFf7mze7xsRDTF16hS+cfKXueDci/nZhf833s1ZMUzCFMdNwHOA+TXXEzGpHH3c4dwx7y5O/vaZ492UFcckXItjHeAWSVcDTwwctL1XzfVOWJ844otcc/3vePjhR3jl3gfw/oPfzpprTOMLX/tvHnp4Ie//xBE8f7NNmPW1o8a7qTFOtnvJ1uy97x7cdvPt/Pj/igB97FHf4rJfXDHOLZvgGtiDVsXsQ2eFS/801HHbl4323qQ4Yihb/uM+492EaKB5C+ZorGUsOny/yjFn9c/+75jrq6LWHrTtyyRtBGxm+xeSVgOm1FlnRERHGpjiqHsUx7uBc4ETy0MbAD+qs86IiI70u/rWI3XPJDwE2Bl4BMD27cCza64zIqJt3RxmJ+kjkm6WdJOksySt0kmb6g7QT9h+cmBH0lQgueWIaJ4u9aAlbQB8ENje9gsp0rodLZhS9yiOyyR9ClhV0quB9wMX1FxnRET7upu6mEoR9xYDqwH3dVJI3T3oQ4EFwI3Ae4CLgMNqrjMion19fdW3Edi+l2IG9R8p5oAstP2zTppU9yiOfuCkcouIaKx2nkkoaSYws+XQrHImNJKeCbwB2Bh4GDhH0gG2z2i3TbUGaEk38tSc80JgDvB52w/WWX9ERGVtBOjWZSmG8CrgLtsLACT9ENgJaFaABn4K9AHfK/f3o8jH/Bk4BXh9zfVHRFTTvUWQ/gi8tJz38TfglRSd0rbVHaBfZXvblv0bJV1ne1tJB9Rcd0REdV26SWj7KknnAtcBS4DrGb63PaK6A/QUSTvavhpA0g4sm0m4pOa6IyKq6+IoDttHAEeMtZy6A/S/Ad+VNA0QxYSVgyWtDnyh5rojIipzX/Ometc9iuMaYCtJa5b7C1tOn11n3RERbWnganY9eWhsGZizcG1ENJb7XXnrlV4+NHaDHtYVEdGeBvagexmgr+9hXRER7WleCrq+FIekKZKWpjVsv6uuuiIixspL+itvvVJbD9p2n6SNJK3cuqJdREQjNbAHXXeK407gCknnA0ufC2/72JrrjYhoSy9v/lVVd4C+o9xWAqbXXFdEROcmWw/a9mfqLD8iolsmTQ9a0nG2PyzpAoZ4gortveqoNyKiY5OoB316+fdXayo/IqKr3MDVgWoJ0LavLV/eavuB1nOStqijzoiIsXADe9CjjoOW9CZJ08vXh0o6W9I2Fcv/taR9Wsr6GHBeZ02NiKhRfxtbj1SZqHKk7Ucl7QTsTrGmxrcrlr8b8HZJ50i6HNgc2LGjlkZE1Mj91bdeqRKgB56QuCdwou0fA0+vUrjt+cDFwMuAGcCpth/roJ0REbVqYoCukoOeL+kE4LXA9pJWpuIUcUm/oHjc+AuB5wHfkXS57Y932uCIiDq4T+PdhKeoEmj3AS4D9rD9V2Ad4NCK5X/T9jtsP2z7RooHJy4c7U0REb02oXrQktZo2b245dhjwBVVCrf9o0H7S4DPtd/MiIh6ub95PeiRUhw3U0wyaW31wL6BDYd7o6TZtneR9CjLT1QRYNtrDPPWiIhx0cRhdsMGaNvP67RQ27uUf2f9jYiYEOyJ1YNeStJ+wCa2j5b0XGDdlskoQ12/1kjl2X6ovWZGRNRrQvWgB0j6JvA0YFfgaOBxinHQO4zwtmtZPh0Cy1IlBjbpsL0REbXob+Aojio96J1sbyvpeih6v+VQu2HZ3hhA0krA/sDGtj8raUNgvbE2OiKi25p4k7DKMLvFZaA1gKS1qT7Z8QTgpcBby/1HgW+228iIiLq5X5W3XqkSoE8AfgA8S9JngNnAlyqW/xLbhwB/ByjHUY/Y+46IGA929a1XRk1x2D5N0rXAq8pDb7F9U8XyF0uawrLe97No5KqrETHZNTHFUXW50SnAYopA286TwI+nWL3u2ZKOAt4MHNZWCyMiemBCDrOT9GngbRSBVsD3JJ1p+wujvdf2mWXv+5Xle/e2fesY2xwR0XV9E3QUxzuAF9t+HKDsCV8PjBqgAWzfBtzWcQsjInpgQvaggfmDrptaHouIWGFMqBy0pK9R5JwfAm6WdEm5/xrgmt40LyKiN3o5OqOqkXrQAyM1bgYubDn+2/qaExExPiZUD9r2d3rZkIiI8dTX384Atd6oMopjU+Ao4AXAKgPHbW9eY7siInqqiSmOKj8yTgFOphgm9zrgbOD7NbYpIqLn+q3K22gkPUPSuZJuk3SrpJd10qYqAXo125cA2L7D9mEUgToiYoVhq/JWwdeBi20/H9ga6Gj+R5Vhdk+UiyXdIem9wL1AFuKPiBVKt1IcktakWJ75wKJcPwk82UlZVQL0R4DVgQ9S5KLXBN7VSWXtWHX9l9ddRUTEUlVSFwMkzQRmthyaZXtW+XpjYAFwsqStKdbH/5DtRe22SW5iZhyYuvIGzWxYRDTOkifvHfMYuavWf1PlmPOS+344bH2StqcYjryz7askfR14xPZ/tdumkSaqnMfyD3xdju03tVtZRERTdbFHeA9wj+2ryv1zgUM7KWikFEcW1o+ISaOdFMdIbP9Z0p8kbWH79xSLxd3SSVkjTVS5tNMGRkRMNF1eLOkDwJnl4wHvBA7qpJCq60FHRKzQuvkkEdtzge3HWk4CdEQEYCbQWhyDSXq67SfqbExExHhZ0sD1oEedSShpR0k3AreX+1tL+kbtLYuI6CGjyluvVJnqfTywJ/AggO0bgFfU2aiIiF7rb2PrlSopjpVs3y0t91Ojr6b2RESMi4mag/6TpB0BS5pCMXxkXr3NiojorV72jKuqEqDfR5Hm2BC4H/hFeSwiYoXRNxF70LYfAPbrQVsiIsZNA594VemJKicxxDR12zOHuDwiYkLqn4g9aIqUxoBVgDcCf6qnORER46OJy2dWSXEs93grSacDs2trUUTEOJioNwkH2xhYt9sNiYgYT/2agCkOSX9lWe9/JeAhOlzbNCKiqZo4uWPEAK1idsrWFM8hBOh3Ux/BEhExBk0cxTHiVO8yGF9ku6/cEpwjYoXUjypvvVJlLY65kl5ce0siIsaR29h6ZaRnEk61vQR4MXCNpDuARYAoOtfb9qiNERG1a2KKY6Qc9NXAtsBePWpLRMS4mWjD7ARg+44etSUiYtz0TbAe9LMkfXS4k7aPraE9ERHjYqL1oKcA06CBE9QjIrpsogXo+bY/27OWRESMowY+knD0HHRExGQw0XrQr+xZKyIixtmEmupt+6FeNiQiYjxNtHHQERGTxkRLcURETBoJ0BERDdXEleASoCMiSA46IqKxJtQojoiIyaS/gUmOBOiICHKTMCKisZrXf06AjogA0oOOiGisJepuH1rSFGAOcK/tPTspo8ozCSMiVng1PJPwQ8CtY2lTAnREBEWKo+o2GknPBfYA/mcsbUqKIyKCrg+zOw74JDB9LIWkBx0RQXspDkkzJc1p2WYOlCNpT+AB29eOtU3pQUdE0N4oDtuzgFnDnN4Z2EvS7sAqwBqSzrB9QLttSg86IgLow5W3kdj+T9vPtT0D2A/4ZSfBGdKDjogAMg46IqKxXMNcQtu/An7V6fsToCMiaGYPOjnoBjtp1jHcd88NzL3+0vFuSjRIPhf16MeVt15JgG6w0047mz323H+8mxENk89FPWqYSThmCdAN9uvZV/HQXx8e72ZEw+RzUY8luPLWK7XnoCVtAGzUWpfty+uuNyKiHXXcJByrWgO0pC8B+wK3sOyJMgaGDNDlbJyZAJqyJiuttHqdzYuIWKqJNwnr7kHvDWxh+4kqF7fOzpm68gbN+3EWESusSdeDBu4EngZUCtAREeOliT3oum8SPg7MlXSipOMHtprrXGGccfoJzL78fLbYfFP+cOccDjpwv/FuUjRAPhf16LMrb70i11iZpHcOddz2qaO9NymOiKhqyZP3aqxlvG2jN1aOOd+7+7wx11dFrSkO26dKWhXY0Pbv66wrImIsmpiDrjXFIen1wFzg4nJ/G0nn11lnREQnuvlElW6pOwd9JLAj8DCA7bnAJjXXGRHRtiZO9a57FMdi2wul5dI1TbxZGhGTXBNTHHUH6JslvQ2YImkz4IPAb2quMyKibb0cnVFV3SmODwBbUoyDPgt4BPhwzXVGRLRt0qU4bD8OfLrcIiIaq4m517rX4riAp67OtxCYA5xo++911h8RUVUTc9B1pzjuBB4DTiq3R4BHgc3L/YiIRph0KQ5gJ9s7tOxfIOka2ztIurnmuiMiKqtzVnWn6g7Q0yRtaPuPAJI2BKaV556sue6IiMr6GpjiqDtAfwyYLekOQMDGwPslrQ6Muh5HRESv9DJ1UVXdozguKsc/P7889PuWG4PH1Vl3REQ7mpjiqP2ZhLafsH0DcEhGbUREU03Gm4Sttu9hXRERbWniMLteBugHelhXRERbJtVUb0lTJH11YN/2a+uqKyJirCZVisN2n6Rd6io/IqKbJt0oDuD6coH+c4BFAwdt/7DmeiMi2tLEURx1B+hVgAeBf245ZiABOiIaZdL1oG0fVGf5ERHdMmlGcUj6pO0vS/oGT13NDtsfrKPeiIhO9bl5C47W1YO+tfx7Tk3lR0R01aTJQdu+oHz5/cGzByWtU0edERFj0cQcdN1Tva+W9NKBHUn/Sp5JGBEN5Db+9Erdozj2B74r6VfA+sDaLD+iIyKiEfq7lOKQ9DzgNGBdintws2x/vZOy6h7FcaOko4DTKZ6ksqvte+qsMyKiE13sGS8BPmb7OknTgWsl/dz2Le0WVPczCb8DbAq8iOIxVz+R9A3bJ9RZb0REu7o1isP2fGB++fpRSbcCGwBtB+i6c9A3Aq+wfZftS4CXANvWXGdERNv67cqbpJmS5rRsM4cqU9IM4MXAVZ20SU0cWgIwdeUNmtmwiGicJU/eq7GWsdmztqscc25fcO2o9UmaBlwGHNXp8hZ1TVQ52/Y+km5k+YkqAmz7RXXUGxHRqW7dJASQ9DTgB8CZY1l7qK4c9IfKv/esqfyIiK7q1k1CSQK+A9xq+9ixlFXXRJWBBPnddZQfEdFtfe7rVlE7A28HbpQ0tzz2KdsXtVtQXSmOR1mW2hjI1ZhlKY416qg3IqJT3bofZ3s2y+LemNTVg55eR7kREXWZjFO9kbSLpIPK1+tI2rjuOiMi2mW78tYrdU9UOYLiad5bACcDKwNnUORoIiIao5ujOLql7rU43kgxSPs6ANv3lVMfIyIaZdIs2N/iSduWZABJq9dcX0RER5q4YH/dOeizJZ0IPEPSu4FfACfVXGdERNsmXQ7a9lclvRp4hCIPfbjtn9dZZ0REJyZjDpoyIP9c0p4JzhHRVE1cl6j2YXYtPtvDuiIi2tKPK2+9UnsPukVXZtZERNShiT3ouqZ6v9T2bwcdfk8ddUVEdMNkGsXxrcEHbF9dU10REWPWzoL9vdLLFEdERGNNmhQHsImk84c7aXuvmuqNiOjIZJpJuAA4pqayIyK6bjL1oB+1fVlNZUdEdN1kmqjyh7EW0I2HQK4oJM20PWu82xHNks9FdzUx5tT2VG9JzwYOAbYsD90MfMv2/bVUuAKTNMf29uPdjmiWfC5WfLUMs5O0M3BNuXtauQFcVZ6LiIhR1JXiOAbY2/b1LcfOl3QecCLwkprqjYhYYdQ1UWWNQcEZANtzgSzY377kGWMo+Vys4GrJQUu6FdjJ9l8HHV8L+I3t53e90oiIFUxdPeivAT+T9E+SppfbbsBPy3MRETGKOkdx7Al8kuVHcXzF9gW1VBgRsYKpbT1o2z+xvavttctt18kYnCX9QdI65evfjHLt9pKO703LohOSLOmYlv2PSzqyZX+mpNvK7WpJu5THz5M0V9L/k7SwfD1X0k5t1P1Y+ff6ks4d5dq9JB3a9hcYjVJXDvrwEU7b9ue6XmlDSfoDsL3tv4yhjKm2l3SvVdEpSX8H5gM72P6LpI8D02wfWf7W+BngX8pz2wI/Ana0/efy/bsBH7e9Zwd1P2Z72hjbn8/SBFJXD3rREBvAwcB/1FTnuJN0QNlrmivpRElTBp0f6AG9UdKlKqwnaZ6k50jaTdJPymuOlHS6pCuA04eoawdJvyvr+oqkm8rjU8r9a8rz7ymP7ybpV5LOLXt3Z0pq3MypCWAJxeiJjwxx7j+ATwz8MLZ9HXAqxYStyiR9ouX795khzs9o+X5/RNJ3y9dbSbpJ0mqSDpT0zfL4KZK+Lekq4MtDlLd7+Zm4VtLxLZ/B1SV9t/xMXy/pDeXxAyX9UNLFkm6X9JQyoztqCdC2jxnYKD7MqwIHAf8LbFJHneNN0j8C+wI7294G6AP2H+pa2+dR9MIOoXjK+REDPaxBXgC8yvZbhzh3MvCelroGHAwstL0DsAPwbkkbl+deDHy4LHcTIJOGOnMCsL+kNQcd3xK4dtCxOSy7DzMqSa8BNgN2BLYBtpO06whv+TrwD5LeyLLPxONDXPdcipFVHx1U3yoUcxNeZ3s74Fktpz8N/NL2jsArgK9IWr08tw3F530rYF9Jz6v6NUZ1ta0HXQ6p+yhFkDoV2HbwsLsVzCuB7YBryo7pqsADI1z/AeAm4Le2zxrmmvNt/23wQUnPAKbbvrI89D1g4Ffm1wAvkvTmcn9Niv/wTwJX276nLGMuMAOYXemri6VsPyLpNOCDwFO+P2P0mnIbmEcwjeL7d/kwbemXdCDwO+BE21cMU+45tvuGOP584E7bd5X7ZwEzW9qyV5nGAVgF2LB8fanthQCSbgE2Av40+pcX7ajrkVdfAd5E0XveyvZjddTTMAJOtf2fyx0s/vMM5blAP7CupJXsIZ+3M5AaQtLJFD3g+4C3jdKOD9i+ZFA7dgOeaDnURx7YMBbHAddR9FoH3ELxQ/qXLce2oxjBNCRJL6HowQIcTvH9+4LtE4d7zxA2Ax4D1h/hmtbP0iXAuhS9+2+O8B4B/2r790O0OZ+lHqgrB/0xig/LYcB9kh4pt0clPVJTnePtUuDNKhaJQtJakjYa6kJJU4HvAm8FbqX4TWNEtg+yvY3t3W0/DDxa/kcB2K/l0kuA90l6WlnX5i2/lkaX2H4IOJsipTTgy8CXJK0NIGkb4ECGeARcSzlXld/XbWyfT/H9e5ekaWUZGwx8poZSplmOB3YF1m75zWmktv9LWd+/Ab+neMDGjPL0vi2XXgJ8YOBehaQXj1Z2dFctP/Vs1zZ8r6ls3yLpMIoJOisBixn+5tCngF/bni3pBoq0yIVtVnkwcJKkfuAyYGF5/H8oUhfXlf+xFgB7t1l2VHMM8O8DO7bPl7QB8BtJBh4FDrA9v2qBtn9W3s+4soyLjwEHMHy67GvACbbnSToY+D9JQ6ZDhqnvb5LeD1wsaRHLFjkD+BzFbwq/Kz/Td7EslRY9UNtElaiXpGkDqSMV413Xs/2hcW5WTEADn6XyB/oJwO22M+O3ASZdT3cFskc5xO4m4OXA58e7QTFhvbu8aXwzxU3ldvLfUaP0oCMiGio96IiIhkqAjohoqAToiIiGSoCOp5DUN3ADUtI5klYbQ1mt64uMuMKapGeUQ77arePIltluox4fdM0pVcYOt1y/dB2MiLolQMdQ/lZOZHghxRTx97aeVKHtz47t821/cYRLngG0HaAjVlQJ0DGaX1MsxjND0u/LNShuAp4n6TWSrpR0XdnTHpj99tpydbTrKKb8Ux5vXWFtXRVrJN9QbjsBXwQ2LXvvXymvG3JlN0mfVrEK4Gxgi9G+CEnvLsu5QdIPBv1W8CpJc8ry9iyvH3JVwEFlbqllqxf+TtJm7f/zRgwvATqGVU5Jfx1wY3loM+BbtrekWNvhMIrV9ralWNfhoypWRzsJeD3FOhTPGab444HLbG8NbEsxBvdQ4I6y9/4JDbOym6TtKKa3bwPsTrFq32h+aHuHsr5bWX6K9oyyjj2Ab5dfw0irAg54L/D1ckXB7YF7KrQjorIscBJDWbWcuABFD/o7FGur3G37t+Xxl1IsW3pFOSV5ZeBKitXR7rJ9O4CkM1i2OlqrfwbeAVCusrZQ0jMHXTPcym7TgfMGltWUdH6Fr+mFkj5PkUaZRrHOxICzy8Wqbpd0Z/k1DLcq4LyW910JfFrScyl+ANxeoR0RlSVAx1D+VvYKlyqD8KLWQ8DPB69VXS4Q1C1Druwm6cMdlHUKsLftG1SsMLhby7nBs7XM8KsCzlh6kf09FYvg7wGMzaNpAAABD0lEQVRcJOk9tltXsosYk6Q4olO/BXaW9A+w9OkbmwO3ATMkbVpeN9TDBqBY/e995XunlKuyPUrROx4w3MpulwN7S1pV0nSKdMpopgPzVazyN/hBCm+RtFLZ5k0oVngbdVVASZtQrKV8PPBj4EUV2hFRWXrQ0RHbC8qe6FmSnl4ePqxcVW0mcKGkxylSJNOHKOJDwKxyBbY+4H22r5R0RTmM7adlHvopK7vZvk7S94EbKFZ5u2aI8gf7L+AqitX9rhrUpj8CVwNrAO+1/XdJVVYF3Ad4u6TFwJ+Boyu0I6KyrMUREdFQSXFERDRUAnREREMlQEdENFQCdEREQyVAR0Q0VAJ0RERDJUBHRDTU/wfMa6BYlNcMJgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"elixir\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20 , random_state=11)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "\n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['elixir-gen','NOT-elixir-gen']); ax.yaxis.set_ticklabels(['elixir-gen' , 'NOT-elixir-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "    auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "    print('ROC AUC: %f' % auc)\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "The number of classes has to be greater than one; got 1 class",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-141-6beb9cbe608f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0msvclassifier\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSVC\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkernel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'linear'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0msvclassifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprobability\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0msvclassifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msvclassifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0my_prob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msvclassifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/sklearn/svm/base.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    148\u001b[0m                          \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'C'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'csr'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m                          accept_large_sparse=False)\n\u001b[0;32m--> 150\u001b[0;31m         \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_targets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m         sample_weight = np.asarray([]\n",
      "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/sklearn/svm/base.py\u001b[0m in \u001b[0;36m_validate_targets\u001b[0;34m(self, y)\u001b[0m\n\u001b[1;32m    523\u001b[0m             raise ValueError(\n\u001b[1;32m    524\u001b[0m                 \u001b[0;34m\"The number of classes has to be greater than one; got %d\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 525\u001b[0;31m                 \" class\" % len(cls))\n\u001b[0m\u001b[1;32m    526\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    527\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: The number of classes has to be greater than one; got 1 class"
     ]
    }
   ],
   "source": [
    "    tool=\"genprog\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=1)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "\n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['genprog-gen','NOT-genprog-gen']); ax.yaxis.set_ticklabels(['genprog-gen' , 'NOT-genprog-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "#     auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "#     print('ROC AUC: %f' % auc)\n",
    "    print(\"genprog label is not good\")\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: -0.105263\n",
      "ROC AUC: 0.500000\n",
      "Accuracy: 0.533333\n",
      "Precision: 0.250000\n",
      "Recall: 0.200000\n",
      "F1 score: 0.222222\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWIAAAEWCAYAAABc752tAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcHUW5xvHfk4U1IeyRVRYNiiiQILLLIlzBiIALKAbh4g2rioheFBRQUYELIrJogMsW4BqEKEZWkcgiREIISwDZkUAkLMoWCMnMe//omnAyzJzpM+mePjnzfP30Z+Z016mqIcd3aqqr3lZEYGZm1RlQdQfMzPo7B2Izs4o5EJuZVcyB2MysYg7EZmYVcyA2M6uYA7EtMklLS/qDpFckXbEI9ewr6YYi+1YFSddK+krV/bDFhwNxPyLpS5KmSnpd0qwUMLYpoOrPAcOBlSLi872tJCIujYhdCujPQiRtLykkTex0fuN0fnLOeo6XNL6nchGxa0Rc1MvuWj/kQNxPSDoSOB34CVnQXBs4G/hMAdW/F3gkIuYXUFdZXgC2lLRSzbmvAI8U1YAy/v+UNS4ifLT4AQwDXgc+X6fMkmSB+rl0nA4sma5tD8wEvgXMBmYBB6RrJwBvA/NSGwcCxwPja+peBwhgUHq9P/AE8BrwJLBvzfnbat63FXAX8Er6ulXNtcnAj4DbUz03ACt387N19P9XwGHp3EDgWeAHwOSasr8AngFeBe4Gtk3nP9np57y3ph8npn68Cbwvnftqun4OcGVN/ScBNwGq+nPho3kO//buH7YElgIm1ilzDLAFsAmwMbA5cGzN9feQBfQ1yILtWZJWiIjjyEbZv4mIIRFxfr2OSFoWOAPYNSKGkgXb6V2UWxH4Yyq7EnAa8MdOI9ovAQcAqwJLAEfVaxu4GNgvff8fwANkv3Rq3UX232BF4DLgCklLRcR1nX7OjWveMwYYCwwFnu5U37eAD0vaX9K2ZP/tvhIRzi1gCzgQ9w8rAS9G/amDfYEfRsTsiHiBbKQ7pub6vHR9XkRcQzYq3KCX/WkHNpK0dETMiogZXZT5FPBoRFwSEfMj4nLgYeDTNWUuiIhHIuJNYAJZAO1WRPwVWFHSBmQB+eIuyoyPiJdSm6eS/aXQ0895YUTMSO+Z16m+OWT/HU8DxgNfi4iZPdRn/YwDcf/wErCypEF1yqzOwqO5p9O5BXV0CuRzgCGNdiQi3gD2Bg4GZkn6o6QP5OhPR5/WqHn9z1705xLgcGAHuvgLQdJRkh5KK0D+TfZXwMo91PlMvYsRMYVsKkZkvzDMFuJA3D/cAcwF9qhT5jmym24d1ubdf7bn9QawTM3r99RejIjrI2JnYDWyUe65OfrT0adne9mnDpcAhwLXpNHqAmnq4DvAF4AVImJ5svlpdXS9mzrrTjNIOoxsZP1cqt9sIQ7E/UBEvEJ2U+osSXtIWkbSYEm7Sjo5FbscOFbSKpJWTuV7XKrVjenAdpLWljQM+G7HBUnDJX0mzRXPJZviaO+ijmuAEWnJ3SBJewMbApN62ScAIuJJ4ONkc+KdDQXmk62wGCTpB8ByNdefB9ZpZGWEpBHAj4Evk01RfEdS3SkU638ciPuJNN95JNkNuBfI/pw+HPhdKvJjYCpwH3A/MC2d601bNwK/SXXdzcLBc0Dqx3PAy2RB8ZAu6ngJGE12s+slspHk6Ih4sTd96lT3bRHR1Wj/euA6siVtTwNvsfC0Q8dmlZckTeupnTQVNB44KSLujYhHge8Bl0haclF+Bmst8s1bM7NqeURsZlYxB2IzswJJ2kDS9JrjVUlH1H2PpybMzMohqWMH58ciovNyzAU8IjYzK89OwOP1gjBAvQX+lZr34hMeqtu7nDzq+1V3wZrQMU9fqp5L1ddIzFlilfUPItvW3mFcRIzroug+ZEtD62raQGxm1qxS0O0q8C4gaQlgd2rW0XfHgdjMDKC9regadwWmRcTzPRV0IDYzA2grPJ32F8kxLQEOxGZmAER0tdO+d9IW/p2Bg/KUdyA2MwNoLy4QpyyDK/VYMHEgNjMDKHBE3CgHYjMzKONmXW4OxGZm4BGxmVnVovhVE7k5EJuZQaE36xrlQGxmBp6aMDOrnG/WmZlVzCNiM7OK+WadmVnFfLPOzKxaEZ4jNjOrlueIzcwq5qkJM7OKeURsZlaxtnmVNe1AbGYGnpowM6ucpybMzCrmEbGZWcUciM3MqhW+WWdmVjHPEZuZVcxTE2ZmFfOI2MysYh4Rm5lVzCNiM7OKzXdieDOzalU4Ih5QWctmZs2kvT3/0QNJy0v6raSHJT0kact65T0iNjODokfEvwCui4jPSVoCWKZeYQdiMzMobNWEpGHAdsD+ABHxNvB2vfd4asLMDLIRcd6jvnWBF4ALJN0j6TxJy9Z7gwOxmRlkqyZyHpLGSppac4ytqWkQMBI4JyI2Bd4Ajq7XtKcmzMwAIhooGuOAcd1cngnMjIgp6fVv6SEQe0RsZgaFrZqIiH8Cz0jaIJ3aCXiw3ns8IjYzg6K3OH8NuDStmHgCOKBeYQdiMzModPlaREwHNstb3oHYzAygra2yph2IzczA2dfMzCrnQGxmVjGnwTQzq1a0519HXDQHYjMz8NSEmVnlvGrCzKxiHhGbmVXMgdi68uTTMznqBz9d8Hrmc7M4/KtjGLP3nhX2yqo2cMnB7Dfh+wxcYhADBg3k4Wv+xi0/v7Lqbi3+Gkj6UzQH4ia27nvX5MqLzgKgra2NHfcYw04f36riXlnV2ubOY/wXT2TenLkMGDSQ/X77Ax6bfC/P3fNY1V1bvLXiiFjS1sDxwHtTOwIiItYrq81WdufU6ay1xmqs/p7hVXfFmsC8OXMBGDBoIAMHD6x0NNcyWnT52vnAN4G7gepuR7aIa2/6C7t94uNVd8OahAaIAyedyArrDGfqxTfy3PTHq+7S4q/CVRNl5iN+JSKujYjZEfFSx1HvDbVZ78+7+PISu7Z4mTdvHpNvm8IuO25bdVesSUR7cN5u3+OMLb7G6puszyoj1qy6S4u9aG/PfRStzBHxzZJOAa4C5nacjIhp3b2hNuv9vBef8N9aya13TuWDI9Zn5RVXqLor1mTmvjqHp//6IOtt/xFeeGRm1d1ZvLXo1MTH0tfanJwB7Fhimy3pmhsns9vO21fdDWsSy6w4lLb5bcx9dQ6DlhzMuttuxB3nTKq6W4u/Vsw1ERE7lFV3fzLnzbe44657OO47X6+6K9Ykhqy6PJ8+7WA0YAAaIB6aNIXH/nxP1d1a/LXiiFjScOAnwOoRsaukDYEtI+L8stpsRcssvRS3Xzuh6m5YE5n98DOcv9sxVXej9cxvzZt1FwLXA6un148AR5TYnplZ70V7/qNgZQbilSNiAtAOEBHz8TI2M2tW7ZH/KFiZN+vekLQS2Q06JG0BvFJie2ZmvVbGsrS8ygzERwJXA+tLuh1YBfhcie2ZmfVeK96si4hpkj4ObEC2vfnvETGvrPbMzBZJKwZiSXt1OjVC0ivA/RExu6x2zcx6pUUTwx8IbAncnF5vT5Z3Yl1JP4yIS0ps28ysIa36zLpBwAcj4nlYsK74YrIdd7cADsRm1jxaNBCv1RGEk9np3MuSPFdsZs2lRVdNTJY0Cbgivf5sOrcs8O8S2zUza1yLjogPA/YCtkmvLwaujIgAnIfCzJpLgYFY0lPAa2Sb2OZHxGb1ype5fC2AK4ErJY2OCKeHMrOmFW2FT03sEBEv5ilY5hbnWj/so3bMzHqnwi3OfRWI1UftmJn1SrRH7qP2aULpGNu5OuAGSXd3ce1d+uopzgf1UTtmZr3TwEi39mlC3dgmIp6VtCpwo6SHI+KW7goXHoi72FHXcX5NgIi4qug2zcwWWYFTxBHxbPo6W9JEYHOy/RNdKmNE/On0dVVgK+DP6fUOwF/JnmFnZtZUYn4xkTgt0R0QEa+l73ehh/tkhQfiiDggdeYGYMOImJVer0aWLN7MrPkUNyIeDkyUBFmMvSwirqv3hrJ31s2qef08sHaJ7ZmZ9VpRuSYi4glg40beU2YgvknS9cDl6fXewJ9KbM/MrPeq2+Fc6oaOw9ONu23TqXERMbGs9szMFkWrZl/rWCHhm3Nm1vxaaUQs6baI2EbSa6Tn1XVcItv5vFzRbZqZLaqYX13bZaya2CZ9HVp03WZmZYkKR8Q9bnGWtJekoen7oyVNkLRJ3gYkrSpp7Y5jUTprZlaa9gaOguXJNXF8Wpi8FbAbcCnwq57eJGl3SY8CTwJ/AZ4Crl2EvpqZlSba8x9FyxOIO56oNxr4dUT8Hlgyx/t+BGwBPBIR6wI7AXf2qpdmZiWrMhDnmSOeJeks4JPAZpKWIF8AnxcRL0kaIGlARNws6fRF6q2ZWUmirbokkXkC8RfIpiR+GRH/krQ6cHSO9/1b0hDgVuBSSbOBN3rfVTOz8lR5s67bQCypdpnZdTXnXgduz1H37sBbwDeALwPLASf0uqdmZiWK9uYcEc8gWwdc27uO10E3eSM61hGT5ZboWEfcUcePJb0MnBIRZy9Kx83MitSUI+KIWKs3Ffa0jljSSmTpMB2IzaxpRFQ3Is71qCRJ+0j6Xvp+TUmjettgRLwEbN/b95uZlaGpl69JOpMsqfuYdGoOOdYR19MpPaaZWeXa25T7KFqeVRNbRcRISfcARMTLaQmbmVnLaNabdR3mSRpAuvGW5ngrnNY2MytelYE4zxzxWcCVwCqSTgBuA04qtVdmZn0sIv9RtB5HxBFxsaS7gU+kU5+PiAeK74qZWXWafWoCYCAwj2x6ItdKCzOzxUlTL1+TdAzZc+dWB9YELpP03bI7ZmbWl9ralPsoWp4R8X7AphExB0DSicA9wE8L742ZWUWqHBHnyr7WqdygdM7MrGU05RyxpJ+TzQm/DMyQdH16vQtwV990z8ysb5SxGiKveiPijpURM4A/1px3cnczazlNOSKOiPP7siNmZlVqa69uQViPc8SS1gdOBDYEluo4HxEjSuyXmVmfqnJqIs+vgAuBC8hyCu8KTAB+U2KfzMz6XHso95GHpIGS7pE0qaeyeQLxMhFxPUBEPB4Rx5IFZDOzlhGh3EdO3wAeylMwTyCem5L+PC7pYEmfBrpM+m5mtrgqMteEpDWBTwHn5Wk7zzribwLLAl8nmyseBvxnnsoXxb2bHFl2E7YYOm62V07aux1TQB15pxwAJI0FxtacGhcR42penw58h5yD1jxJf6akb1/jneTwZmYtpZFVEynojuvqmqTRwOyIuFvS9nnqq7ehYyLvPPyzq47slacBM7PFQYGLJrYGdpe0G9lKs+UkjY+IL3f3hnoj4jOL65eZWXNrZGqinoj4LvBdgDQiPqpeEIb6GzpuKqRXZmaLgWZP+mNm1vLKeP5bREwGJvdUzoHYzAwIFoMRsaQlI2JumZ0xM6vK/CZ/Qsfmku4HHk2vN5b0y9J7ZmbWhwLlPoqWZ+HcGcBo4CWAiLgX2KHwnpiZVai9gaNoeaYmBkTE09JCvwXaSuiLmVllmn2O+BlJmwMhaSDwNeCRcrtlZta3yhjp5pUnEB9CNj2xNvA88Kd0zsysZbQ184g4ImYD+/RBX8zMKlPhk5JyPaHjXLrYhh0RY7sobma2WGpv5hEx2VREh6WAPYFnyumOmVk1KnxSUq6piYUeiyTpEuC20npkZlaBZr9Z19m6wPCiO2JmVqV2NfHUhKR/8c6ofQDwMnB0mZ0yM+trVW6OqBuIle3i2Bh4Np1qj6jyodNmZuWoctVE3S3OKeheExFt6XAQNrOW1I5yH0XLk2tiuqRNC2/ZzKyJRANH0eo9s25QRMwHNgXukvQ48AYgssHyyBL6Y2ZWiWbd0PE3YCSwex/1xcysMs26fE0AEfF4H/XFzKwybU06Il5F0pHdXYyI00roj5lZJZp1RDwQGAIVbsA2M+sjzRqIZ0XED/usJ2ZmFarwkXU9zxGbmfUHzToi3qnPemFmVrGm3OIcES/3ZUfMzKrUrOuIzcz6jWadmjAz6zcciM3MKlZUDglJSwG3AEuSxdjfRsRx9d7jQGxmRqFzxHOBHSPidUmDgdskXRsRd3b3BgdiMzOKWzWR0gW/nl4OTkfdAXeeNJhmZi2vnch9SBoraWrNsdBT7SUNlDQdmA3cGBFT6rXtEbGZGY3drIuIccC4OtfbgE0kLQ9MlLRRRDzQXXmPiM3MKCcxfET8G7gZ+GS9cg7EZmZkI+K8Rz2SVkkjYSQtDewMPFzvPZ6aMDMD5quwhyCtBlwkaSDZYHdCREyq9wYHYjMziltHHBH3kT1iLjcHYjMzvLPOzKxy7aU8nzkfB2IzM4qbmugNB2IzMzw1YWZWuTZPTZiZVcsjYjOzioVHxGZm1apyROwtzs1uwAA2vO403nfhMVX3xJrEueNO5bmZ9zL9npuq7kpLaST7WtEciJvc8ANH8+ZjM6vuhjWRiy+ewKdG71t1N1pOGUl/8nIgbmKDV1uJYTttxouX3Vh1V6yJ3HrbFF7+17+r7kbLmU/kPormQNzE1jr+QGaeeBFElUvNzfqHaOB/RSs1EEsaIelcSTdI+nPHUaf8gqz3V73xVJlda3rDdtqM+S++wpz7H6+6K2b9QlFpMHuj7FUTVwC/As4lxyOharPeT11zj349DBzy0Q+w/C4fZdiOoxiw5GAGDF2Gdc84gie/fnrVXTNrSa28fG1+RJxTchst6dmfjefZn40HYOiWGzH8oM84CJuVqJWXr/1B0qGSVpO0YsdRcptmLW38JWdx2y1Xs8GI9XnqiakcsP8+VXepJbRF5D6KVvaI+Cvp67drzgWwXsnttpTX7niA1+7o9rmD1s98ecxhVXehJbVsGsyIWLfM+s3MilLlHHHZqyaWkXSspHHp9fsljS6zTTOz3qhy1UTZc8QXAG8DW6XXzwI/LrlNM7OGtfIW5/Uj4mRgHkBEzAFUcptmZg2rckNH2Tfr3pa0NGl7tqT1gbklt2lm1rAyVkPkVXYgPg64DlhL0qXA1sD+JbdpZtawVl41caOkacAWZFMS34iIF8ts08ysN1r2CR2SRqZvZ6Wva0saBjwdEfPLbNvMrBGtvMX5bGAkcB/ZiHgjYAYwTNIhEXFDye2bmeVS5dRE2asmngM2jYjNImIUsCnwBLAzcHLJbZuZ5RYRuY+ilR2IR0TEjI4XEfEg8IGIeKLkds3MGtJG5D7qkbSWpJslPShphqRv9NR22VMTMySdA/xfer038KCkJUlri83MmkGBUxPzgW9FxDRJQ4G7Jd2YBqJdKjsQ7w8cChyRXt8OHEUWhHcouW0zs9yKmnKIiFmkBQoR8Zqkh4A1gGoCcUS8CZwKnCppZERMq7n8epltm5k1ooybdZLWIbs3NqVeub58Zt15fdiWmVlDGtniXPtYt3SM7VyfpCHAlcAREfFqvbbLnpqo5RwTZta0GtniXPtYt65IGkwWhC+NiKt6qq8vA/EJfdiWmVlDipqakCTgfOChiDgtz3tKCcQ1O+pq/aPjfKe5YjOzyhU4R7w1MAa4X9L0dO57EXFNd28oa0R8avq6FLAZcC/Z1MRHgKnAliW1a2bWKwWumriNBqdiS7lZFxE7RMQOZEs4RnbaWfdsGW2amS2KKhPDlz1HvEFE3N/xIiIekPTBkts0M2tYKyf9uU/SecD49HpfsgRAZmZNpS2qS4RZdiA+ADgE6NhrfQtwTsltmpk1rIxkPnmVvbPuLeDn6TAza1ot94QOSRMi4guS7od3/3QR8ZEy2jUz661WnCPumIoYXVL9ZmaFam+1qYmUfYiIeLqM+s3MilbliLjUpD+StpB0l6TXJb0tqU1S3eQXZmZVaIv23EfRyl41cSawD3AF2Q67/YARJbdpZtawKqcmSk+DGRGPAQMjoi0iLgA+WXabZmaNaiQNZtHKHhHPkbQEMF3SyWRbnvsyB7KZWS6tPCIeAwwEDgfeANYCPltym2ZmDWvZEXHNqok3cT5iM2tibdFWWdtVbOgI4GXg9Ij4fRntm5k1qhW3OPe0oWNl4FLAgdjMmkLLbXHuakOHpNERMSm9fFrSvmW0bWbWG1WOiPtyBcMPa19ExN192LaZWV3tEbmPovkpzmZmtGDSH0lbRMSdnU4fVEZbZmZFqDIxfFlTE2d3PhERfyupLTOzRRYRuY+i9eXUhJlZ02q5NJjAepKu7u5iROxeUrtmZr3SiuuIXwBOLaluM7PCtdw6YuC1iPhLSXWbmRWuFUfET5VUr5lZKapcNVHWzrq9JK0KHAZ8KJ2eAZwdEc+X0aaZ2aJouTSYkrYG7kovL04HwJR0zcysqbTi8rVTgT0i4p6ac1dLmgj8GvhYSe2amfVKkTvrJP0vWdKz2RGxUU/ly9rQsVynIAxAREwHhpbUpplZrxU8Ir6QBh4LV9aIWJJWiIh/dTq5In5Ukpk1oSLniCPiFknr5C1fViD+OXCDpKOAaencKOCkdK1Hm838nZMEJZLGRsS4qvvRDOZX3YEm4s9Fsea//WzumCNpLDC25tS4Rfm3UFlr5ySNBr7DwqsmTomIP5TSYAuTNDUiNqu6H9Zc/LlobmlEPCnPHHFpuSZSEvhJPRY0M+vnykqD+YM6lyMiflRGu2Zmi6Oybpy90cUBcCDw3yW12co8D2hd8eeiSUm6HLgD2EDSTEkH1i1f9v5qSUPJHiZ6IDABODUiZpfaqJnZYqS0OeK0VO1IYF/gImBk5+VsZmZW3hzxKcBeZH86fTgiXi+jHTOzVlDWHPG3gNWBY4HnJL2ajtckvVpSm01H0l97c13ShZI+V06vrCiSQtKpNa+PknR8zeuxkh5Ox98kbZPOT5Q0XdJjkl5J30+XtFWn+leX9Nse+tBtGUmTJXl522KgrOxr3j0HRMRWi3Ldmt5cYC9JP42IF2svpHX0BwHbRMSLkkYCv5O0eUTsmcpsDxwVEaO7qjwingPq/kLOU8aanwNmiSS9LmmIpJskTZN0v6TP1F5PXyXpTEl/l/QnYNVu6hsg6ew0wrpR0jUdI2dJoyT9RdLdkq6XtFo6P1nSSWlE9oikbfvgR+8v5pNNv32zi2v/DXy7I0BHxDSyeyWH5a1c0jqSHqj5/tb0OZrWMXruVGZpSf8n6aGUYGvpbupdRtIESQ+m0fmUjpGzpF0k3ZHauELSkHT+KUkn1HyOP5D357CeORCX7y1gz4gYCewAnCqp81bKPYENgA2B/YDuRsp7AeukcmOALQEkDQZ+CXwuIkYB/wucWPO+QRGxOXAEcFwBP5O94yxgX0nDOp3/EHB3p3NTeWenaaNmAzunz9HewBldlDkEmBMRHyT7dx7VTV2HAv+KiA2B73eUk7Qy2XTiJ1I7U8luuHd4MZ0/Bziqlz+HdcFPcS6fgJ9I2g5oB9YAhgP/rCmzHXB5RLSRzan/uZu6tgGuiIh24J+Sbk7nNwA2Am5MMX4gMKvmfVelr3eTBXIrSES8Kuli4OvAmyU2NRg4U9ImQBswoosy25ECdETcJ+m+buraBvhFKvdATbktyH7J354+R0uQrYXtUPs52qv3P4p15kBcvn2BVYBRETFP0lPAUnneKOljZPmbAertVhQwIyK27Ob63PS1Df+bl+F0suRWF9Sce5BspFn7S3UUWc6VLnXx710bSL8JPA9sTPaX7Ft5OydpT975S+ir9YoCN0bEF7u57s9RSTw1Ub5hZMmh50naAXhvF2VuAfaWNDDN7e4AEBFTImKTdFwN3A58Ns0VDwe2T+//O7CKpAVTFZJ6+yewNSgiXibbrFS7e+pk4CRJKwGkkez+wNl16un8711rGDAr/TU0huyvns5uAb6U2tsI+Eiqd2JNvVPJPkdfSOU2BD6c3n8nsLWk96Vry0rqauRtBfNvtXIFcCnwB0n3k825PdxFuYnAjmSjqH+w8J+Dta4EdkrlniEbhb0SEW+nm3ZnpLnKQWSjtG5HX1a4U4HDO15ExNWS1gD+KimA14AvR8Ss7iroRsfW17OBKyXtB1zHO2kDap0DXCDpIeAh3j1H3eFs4CJJD5J9HmeQfY5ekLQ/cLmkJVPZY4FHGuyzNaj0Lc79VRoJTYuIrkbAi1LvkIh4PdX/N2DriPhnT++zxY+kUcBpEfHxgusdCAyOiLckrQ/8CdggIt4ush3LzyPiEkhaHZgM/E8J1U+StDzZjZQfOQi3prSc7DLg6BKqXwa4Oa22EXCog3C1PCI2M6uYb9aZmVXMgdjMrGIOxGZmFXMgtneR1KYsG9gDKd/AMotQ1/aSJqXvd5fU7c0nSctLOrQXbRyv7Inhuc53KtNQprva3A5mRXEgtq68mRb/bwS8DRxcezElKWr4sxMRV0fEz+oUWZ4sD4JZv+JAbD25FXhfGgn+PeVVeABYq06mrk8qyxA3jZqcBJL2l3Rm+n54yvx1bzq2An4GrJ9G46ekct+WdJek+ySdUFPXMcqyyd1GlmujLkn/leq5V9KVnUb5n5A0NdU3OpUfKOmUmrYP6qLODynLajc9lXl/4/95zRyIrQ5Jg4BdgfvTqfcDZ0fEh8h2dr0rU5ekpYBzgU+T5VZ4TzfVnwH8JSI2BkaS7e46Gng8jca/LWmX1ObmwCbAKEnbpY0O+6RzuwEfzfHjXBURH03tPcTC25HXSW18CvhV+hkOJNtt9tFU/39JWrdTnQcDv4iITYDNgJk5+mH2Lt7QYV1ZWtL09P2twPlkT1x5OiLuTOe7y9T1AeDJiHgUQNJ4YGwXbexIlvKTlHXuFUkrdCqzSzruSa+HkAXmocDEiJiT2uicl6ErG0n6Mdn0xxDg+pprE1IOh0clPZF+hl2Aj9TMHw9Lbddu970DOEbSmmSB/tEc/TB7Fwdi68qbaZS3QAq2tfkNuszUlZLbFEXATyPi1wudlI7oRV0XAntExL0pn8L2Ndc672qK1PbXIqI2YCNpnQWFIi6TNIVsJH2NpIMiorsUpmbd8tSE9VZ3mboeBtZJOQwAukupeBNZIvOO+dhhZIlxhtaUuR74z5q55zUkrUqWZWwPZU+kGEo2DdKTocCstK13307XPq8so936wHpk2eyuBw5J5ZE0QtKytW+StB7wREScAfyelO3MrFEeEVuvdJepKyIekTQW+KOAea5EAAAAl0lEQVSkOWRTG0O7qOIbwDhJB5Lltz0kIu6QdHtaHnZtmif+IHBHGpG/TpbBbJqk3wD3kj254q4cXf4+MAV4IX2t7dM/yBIoLQccnJLhnEc2dzxNWeMvAHt0qvMLwBhJ88gS/f8kRz/M3sW5JszMKuapCTOzijkQm5lVzIHYzKxiDsRmZhVzIDYzq5gDsZlZxRyIzcwq9v+GzpYdqB/wjwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"jaid\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20 , random_state=1)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['jaid-gen','NOT-jaid-gen']); ax.yaxis.set_ticklabels(['jaid-gen' , 'NOT-jaid-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "    auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "    print('ROC AUC: %f' % auc)\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: 0.634146\n",
      "ROC AUC: 0.961538\n",
      "Accuracy: 0.933333\n",
      "Precision: 1.000000\n",
      "Recall: 0.500000\n",
      "F1 score: 0.666667\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEWCAYAAAB7QRxFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmYXFWZx/HvLxBZkrCEHQJGdkVIgBAREMPoRMAIiMhi2HEiiCiD4qAim8u4PCD7EoYdwgCDYZFdFFlkCyEhYREkgBAiSwJhFZLud/64p0ml0sut7rpddTu/D899UnXq1jmnu5q3T5977nsUEZiZWXn1a3QHzMysZxzIzcxKzoHczKzkHMjNzErOgdzMrOQcyM3MSs6B3HpM0jKSbpQ0V9I1PahnrKTb69m3RpB0i6QDGt0PW3w4kC9GJH1D0iRJ70ialQLOdnWoeg9gNWCliPh6dyuJiCsiYnQd+rMQSaMkhaSJVeXDUvldOes5QdLlXZ0XETtFxCXd7K5ZzRzIFxOSjgJOBX5JFnTXAc4Gdq1D9R8Hno6I+XWoqyivAZ+VtFJF2QHA0/VqQBn/P2W9zj90iwFJywMnAYdHxO8j4t2ImBcRN0bE0emcpSSdKunldJwqaan02ihJL0n6vqRX02j+oPTaicBxwF5ppH9I9chV0tA08l0yPT9Q0gxJb0t6TtLYivJ7K963jaSH05TNw5K2qXjtLkk/k3Rfqud2SSt38m34ELgO2Du9fwlgL+CKqu/VaZJelPSWpEckfS6V7wj8uOLrnFrRj19Iug94D1g3lX0zvX6OpGsr6v+1pDslKfcHaNYFB/LFw2eBpYGJnZzzE2BrYDgwDBgJHFvx+urA8sBawCHAWZJWjIjjyUb5V0XEwIi4oLOOSBoAnA7sFBGDgG2AKe2cNxi4KZ27EnAKcFPViPobwEHAqsDHgB901jZwKbB/evwlYDrwctU5D5N9DwYDE4BrJC0dEbdWfZ3DKt6zHzAOGAS8UFXf94FN0y+pz5F97w4I58awOnIgXzysBLzexdTHWOCkiHg1Il4DTiQLUG3mpdfnRcTNwDvARt3sTyvwaUnLRMSsiHi8nXO+DDwTEZdFxPyIuBJ4CvhKxTkXRcTTEfE+cDVZAO5QRPwVGCxpI7KAfmk751weEbNTmycDS9H113lxRDye3jOvqr73yL6PpwCXA0dExEtd1GdWEwfyxcNsYOW2qY0OrMnCo8kXUtlHdVT9IngPGFhrRyLiXbIpjUOBWZJukrRxjv609Wmtiuf/7EZ/LgO+A+xAO3+hSPqBpCfTdM6bZH+FdDZlA/BiZy9GxIPADEBkv3DM6sqBfPFwP/ABsFsn57xMdtGyzTosOu2Q17vAshXPV698MSJui4h/B9YgG2Wfn6M/bX2a2c0+tbkM+DZwcxotfyRNffwQ2BNYMSJWAOaSBWCAjqZDOp0mkXQ42cj+5VS/WV05kC8GImIu2QXJsyTtJmlZSf0l7STpN+m0K4FjJa2SLhoeRzYV0B1TgO0lrZMutP6o7QVJq0naNc2Vf0A2RdPaTh03AxumJZNLStoL+BTwh272CYCIeA74PNk1gWqDgPlkK1yWlHQcsFzF668AQ2tZmSJpQ+DnwL5kUyw/lNTpFJBZrRzIFxNpvvcosguYr5FNB3yHbCUHZMFmEvAYMA2YnMq609YdwFWprkdYOPj2S/14GZhDFlQPa6eO2cAYsouFs8lGsmMi4vXu9Kmq7nsjor2/Nm4DbiVbkvgC8C8WnjZpu9lptqTJXbWTprIuB34dEVMj4hmylS+Xta0IMqsH+eK5mVm5eURuZlZyDuRmZiXnQG5mVnIO5GZmJdfZDSINNe/1Gb4Ka4tYZs3PNboL1oTmfzizx7lraok5/Vdet6ly5XhEbmZWck07Ijcz61WtLY3uQbc5kJuZAbQ0czr9zjmQm5kBEe1liigHB3IzM4DW8gZyX+w0MwOI1vxHFyRdmHbTml5R9ltJT0l6TNJESSt08N7nJU2TNEXSpDxddyA3M4PsYmfeo2sXAztWld0BfDoiNiNLzPaj6jdV2CEihkfEiDyNOZCbmUFdR+QRcTdZds/KstsrNmd5ABhSr647kJuZAdEyP/chaZykSRXHuBqbOxi4paOuALenzb9z1euLnWZmUNPFzogYD4zvTjOSfkK2gckVHZyyXUTMlLQqcIekp9IIv0MekZuZQV2nVjoi6UCyDVPGRgebQUTEzPTvq2T7yo7sql4HcjMzqPfFzkVI2pFsp6tdqveLrThngKRBbY+B0cD09s6t5EBuZgb1Xn54Jdmm5xtJeknSIcCZZPvC3pGWFp6bzl1T0s3prasB90qaCjwE3BQRt3bVnufIzcygrrfoR8Q+7RRf0MG5LwM7p8czgGG1tudAbmYGpb6z04HczAyIcPZDM7Nyc9IsM7OS89SKmVnJeURuZlZyLfMa3YNucyA3MwNPrZiZlZ6nVszMSs4jcjOzknMgNzMrt/DFTjOzkvMcuZlZyXlqxcys5DwiNzMrOY/IzcxKziNyM7OSm1+/jSV6mwO5mRl4RG5mVnqeIzczKzmPyM3MSs4jcjOzkvOI3Mys5Lxqxcys5CIa3YNucyA3MwPPkZuZlZ4DuZlZyflip5lZybW0NLoH3eZAbmYGnloxMys9B3Izs5LzHLmZWblFa3nXkfdrdAfMzJpCa2v+owuSLpT0qqTpFWWDJd0h6Zn074odvPeAdM4zkg7I03UHcjMzyFat5D26djGwY1XZMcCdEbEBcGd6vhBJg4Hjgc8AI4HjOwr4lRzIzcygriPyiLgbmFNVvCtwSXp8CbBbO2/9EnBHRMyJiDeAO1j0F8IiPEduZga9sWpltYiYlR7/E1itnXPWAl6seP5SKuuUR+RN5thfnsL2X96b3fY99KOyM8Zfylf3P4yvHXA4/3Hkj3n1tdkN7KE1gy+NHsXj0+/mqSfu5YdHH97o7vQNEbkPSeMkTao4xtXWVARQt6urDuRNZred/51zT/n5QmUHjf0aEy89h2svOYvPb/sZzrloQoN6Z82gX79+nH7aLxjzlX3ZdNgO7LXXbnzykxs0ulvlV8PUSkSMj4gRFcf4HC28ImkNgPTvq+2cMxNYu+L5kFTWKQfyJjNi+KYsv9yghcoGDhjw0eP33/8XUm/3yprJyK0259lnn+e55/7BvHnzuPrq69nlK19qdLfKrzXyH91zA9C2CuUA4Pp2zrkNGC1pxXSRc3Qq61Thc+SSliCbC/qorYj4R9Ht9jWnnXcxN9x6J4MGDODCM37V6O5YA6251uq8+NLLHz1/aeYsRm61eQN71EfUMdeKpCuBUcDKkl4iW4nyK+BqSYcALwB7pnNHAIdGxDcjYo6knwEPp6pOiojqi6aLKHRELukI4BWyK683peMPnZz/0bzT/1x6ZZFdK53vfetA7px4GV8evQMTrr2x0d0x63OitTX30WVdEftExBoR0T8ihkTEBRExOyK+EBEbRMQX2wJ0REyKiG9WvPfCiFg/HRfl6XvRUyvfAzaKiE0iYtN0bNbRyZXzTt/cf5+Cu1ZOY0bvwB/vuq/R3bAGennmP1l7yJofPR+y1hq8/PI/G9ijPqL4qZXCFB3IXwTmFtxGn/fCiwuudfzpnvv5xMeHNLA31mgPT5rC+ut/gqFD16Z///7sueeu3PiH2xvdrfKL1vxHkyl6jnwGcJekm4AP2goj4pSC2y2to4//FQ8/+hhvvvkWX9htX759yH7cc//DPP+Pl1A/sebqq3Lc0Uc0upvWQC0tLXzvyGO5+aYJLNGvHxdfchVPPPF0o7tVfk040s5LUeCGo5KOb688Ik7s6r3zXp9R3u+qFWaZNT/X6C5YE5r/4cwer+V697i9c8ecASf9b1OtHSt0RN4WsCUtGxHvFdmWmVmPNOGUSV5Fr1r5rKQngKfS82GSzi6yTTOzbvHFzg6dSpYEZjZAREwFti+4TTOzmtVz+WFvK/yGoIh4UQvfiljeHU7NrO9qwpF2XkUH8hclbQOEpP5k68qfLLhNM7PaOZB36FDgNLI0jDOB2wGnajOz5lPHW/R7W9GrVl4HxhbZhplZPZR5z85CA7mk09spngtMioj2Mn+ZmTVGiQN50atWlgaGA8+kYzOy/LqHSDq14LbNzPKr41Zvva3oOfLNgG0jogVA0jnAPcB2wLSC2zYzy6/EI/KiA/mKwEAWJM4aAAyOiBZJH3T8NjOzXuZA3qHfAFMk3QWI7GagX0oaAPyx4LbNzHKLluabMsmr6FUrF0i6GRiZin4cEW1bmxxdZNtmZjUp8Yi88D07I2JWWqGyeUUQNzNrKtEauY9m05ubL+/Si22ZmdWmxEmzCs+1UqGp8veamS2kvFPkvRrIt+zFtszMahLzyxvJCwnkks4AFvn7oy0LYkR8t4h2zcy6rbxxvLAR+aSC6jUzK0QzXsTMq5BAHhGXVD6XNDCVv1NEe2ZmPeYRefskfRq4DBicPdVrwP4R8XiR7ZqZ1coj8o6NB46KiD8DSBoFnA9sU3C7Zma18Yi8QwPagjhARNyVbs83M2sqMb/RPei+ogP5DEk/JZteAdgXmFFwm2ZmNYsSj8i7vLNT0u6SBqXHx0i6WtLwnPUfDKwC/D4dq6QyM7Pm0lrD0WTyjMhPiIjfp02UdwZOBs4Ftu7qjRHxBvBdScsDrRHxdo96a2ZWkD49IgfadiQdA5yXEmAtladySVtJmgZMBaZJmirJd3iaWdOJ1vxHs8kzIp8l6SxgR2CEpI+RP9nWBcC3I+IeAEnbAReR7RxkZtY0oqW86aDyBOQ9gb8AX05TJSsDx+Ssv6UtiANExL1Aia8Nm1lfVeYReYeBXNJykpZL59wKvJyevwPcl7P+v0g6T9IoSZ+XdDZwl6QtJG3R496bmdVJtCr30RlJG0maUnG8JenIqnNGSZpbcc5xPel7Z1Mrj5MlvqrsddvzANbJUf+w9O/xVeWbpzr+LV83zcyKVa+RdkT8DRgOIGkJYCYwsZ1T74mIMfVos8NAHhFr97TyiNihp3WYmfWGiELmyL8APBsRLxRReZtcFy0l7S3px+nxkLwrTyQtL+kUSZPScXJaimhm1lRqmSOXNK4irk2SNK6DavcGruzgtc+mlXy3SNqkJ33vctWKpDOB/sD2wC+B98jWkW+Vo/4LgelkF0wB9iNbtbJ7dzprZlaU1hpWrUTEeLJcUh1KK/x2AX7UzsuTgY9HxDuSdgauAzbI39uF5Vl+uE1EbCHpUYCImJM6mMd6EfG1iucnSppScy/NzArW1UXMbtgJmBwRryzSVsRbFY9vlnS2pJUj4vXuNJRnamWepH6kHX8krUT+m1TfT2vHSe/dFni/5l6amRWsXqtWKuxDB9MqklZX2jJN0kiyWDy7u33PMyI/C7gWWEXSiWTTJCfmrP8w4JKKefE3gANq7qWZWcGijunIU5bXfwe+VVF2aNZOnAvsARwmaT7Z4HbviO73QHnemybiv5ie3hkR03NVLi1F1uH1gBWAuUBExEldvXfe6zPKm+XdCrPMmp9rdBesCc3/cGaP50VmbDo6d8xZd9rtTXUbaN40tksA88imV/Leng9wPfAm2cT+zNq6ZmbWewpaftgr8qxa+QnwDbIF7QImSLoiIv47R/1DImLHHvbRzKxwLSXOtZJnRL4/sHlEvAcg6RfAo0CeQP5XSZtGxLQe9NHMrHB9ekQOzKo6b8lUlsd2wIGSngM+IN3eHxHOfmhmTaWA5Ye9psNALul3ZHPic4DHJd2Wno8GHs5Z/0497qGZWS+o56qV3tbZiLxtZcrjwE0V5Q/krbzo/AJmZvXSJ0fkEXFBb3bEzKyRWlprWZDXXPKsWlkP+AXwKWDptvKI2LDAfpmZ9aoyT63k+RV0MVmiK5HNeV8NXFVgn8zMel1rKPfRbPIE8mUj4jaAiHg2Io7FFzHNrI+JUO6j2eRZfvhBSpr1bMoVMBMYVGy3zMx6V5mnVvIE8v8EBgDfJZsrXx44uMhOAay34a5FN2EltObAwY3ugvVRzThlkleXgTwiHkwP3ybbGMLMrM/pk6tWJE0k5SBvT0R4lx8z6zNKPLPS6Yj8zF7rhZlZg/XJqZWIuLM3O2Jm1kjNuBolr7z5yM3M+rS8+1c2IwdyMzMgWAxG5JKWiogPiuyMmVmjzC/x1EqX620kjZQ0DXgmPR8m6YzCe2Zm1osC5T6aTZ6Fk6cDY4DZABExFdihyE6ZmfW21hqOZpNnaqVfRLwgLfRbqKWg/piZNUQzjrTzyhPIX5Q0EghJSwBHAE8X2y0zs97VjCPtvPIE8sPIplfWAV4B/pjKzMz6jJa+PCKPiFeBvXuhL2ZmDVPind5y7RB0Pu2kIYiIcYX0yMysAVr78oicbCqlzdLAV4EXi+mOmVlj9NWkWQBExELbukm6DLi3sB6ZmTVAX7/YWe0TwGr17oiZWSO1qg9PrUh6gwV/dfQD5gDHFNkpM7PeVuabYzoN5MruAhpGtk8nQGtEmXe2MzNrX5lXrXR6i34K2jdHREs6HMTNrE9qRbmPZpMn18oUSZsX3hMzswaKGo5m09menUtGxHxgc+BhSc8C7wIiG6xv0Ut9NDMrXD2nViQ9T7ZhfQswPyJGVL0u4DRgZ+A94MCImNzd9jqbI38I2ALYpbuVm5mVRQHLD3eIiNc7eG0nYIN0fAY4J/3bLZ0FcgFExLPdrdzMrCxaenfqe1fg0nTd8QFJK0haIyJmdaeyzgL5KpKO6ujFiDilOw2amTWjWkbkksYBlWlKxkfE+IrnAdwuKYDzql4DWIuF75B/KZXVPZAvAQyEJrxEa2ZWZ7UE8hSYq4Nzpe0iYqakVYE7JD0VEXf3rIcd6yyQz4qIk4pq2MysmdRzy86ImJn+fVXSRGAkUBnIZwJrVzwfwoL7dWrW2fJDj8TNbLFRr63eJA2QNKjtMTAamF512g3A/spsDczt7vw4dD4i/0J3KzUzK5s63qK/GjAxbY+5JDAhIm6VdChARJwL3Ey29PDvZMsPD+pJgx0G8oiY05OKzczKpF7ryCNiBllqk+rycyseB3B4fVrsXvZDM7M+Z3FLY2tm1uc4kJuZlVwz5lDJy4HczIxyp7F1IDczow9vLGFmtrhoLfHkigO5mRm+2GlmVnrlHY87kJuZAR6Rm5mV3nyVd0zuQG5mhqdWzMxKz1MrZmYl5+WHZmYlV94w7kBuZgZ4asXMrPRaSjwmdyA3M8MjcjOz0guPyM3Myq3MI/J+je6Adey3Z5zE5L/dxR33/b7RXbEm4p+LYrQSuY9m40DexK6ZcD37f/2wRnfDmox/LooRNRzNxoG8iT10/yO8+cbcRnfDmox/Looxn8h9NBvPkZuZUe6LnYWOyCXtLukZSXMlvSXpbUlvdXL+OEmTJE1654M5RXbNzGwhrTUczaboqZXfALtExPIRsVxEDIqI5To6OSLGR8SIiBgxcKnBBXfNzGyBqOG/ZlP01MorEfFkwW2YmfVYM4608yp6RD5J0lWS9knTLLtL2r3gNvuMM87/Ndfddjnrrj+UB6f/kb32/Wqju2RNwD8XxWiJyH00G0WBnZJ0UTvFEREHd/XedQZv2nzfLTNrSv+YM009reMbH/9q7pgz4YWJPW6vngqdWomIg4qs38ysXppx7juvoletbCjpTknT0/PNJB1bZJtmZt3hVSsdOx/4ETAPICIeA/YuuE0zs5qV+Rb9oletLBsRD0kLTSfNL7hNM7OalXlqpehA/rqk9UjpCSTtAcwquE0zs5o142qUvIqeWjkcOA/YWNJM4EjA2X7MrOnUa2pF0tqS/izpCUmPS/peO+eMSne8T0nHcT3pe9GrVmYAX5Q0AOgXEW8X2Z6ZWXfV8SLmfOD7ETFZ0iDgEUl3RMQTVefdExFj6tFgoYFc0lFVzwHmAo9ExJQi2zYzq0W95sgjYhZpCjki3pb0JLAWUB3I66boqZURwKFkX8RawLeAHYHzJf2w4LbNzHKrZWqlMsFfOsa1V6ekocDmwIPtvPxZSVMl3SJpk570veiLnUOALSLiHQBJxwM3AdsDj5Al1TIza7ha7nKPiPHA+M7OkTQQuBY4MiKqs75OBj4eEe9I2hm4Dtigth4vUPSIfFXgg4rn84DVIuL9qnIzs4ZqIXIfXZHUnyyIXxERi+zJFxFvtQ1wI+JmoL+klbvb96JH5FcAD0q6HhAwBpiQLn4WNl9kZlaret3oo+xi4AXAkxFxSgfnrE6WHTYkjSQbVM/ubptFr1r5maRbgG1T0aERMSk9Hltk22ZmtahjAsFtgf2AaZLaFnX8GFgntXMusAdwmKT5wPvA3tGDDhS+1VsK3JMkjasI4mZmTaVeI/KIuJdsBqKzc84EzqxLg/Tu5suH9mJbZmY18Q5B+TRV/l4zs0plvkW/NwP5V3qxLTOzmjRjVsO8Cgnk1Xd0VpQD0NGVXDOzRnEgX9Sgguo1MytEkdteFq2QQB4RJxZRr5lZUco8Ii96q7chkiZKejUd10oaUmSbZmbdUeZVK0UvP7wIuAFYMx03pjIzs6bSEq25j2ZTdCBfJSIuioj56bgYWKXgNs3MahYRuY9mU3Qgny1pX0lLpGNfepBPwMysKGXefLnoQH4wsCfwz3TsARxUcJtmZjUr8xx50UmzXgB2KbINM7N6aG3CKZO8il61sq6kGyW9llatXC9p3SLbNDPrjjKPyIueWpkAXA2sQbZq5RrgyoLbNDOrmVetdGzZiLisYtXK5cDSBbdpZlaz1ojcR7MpOmnWLZKOAf4XCGAv4GZJgwEiYk7B7ZuZ5dKMUyZ5FR3I90z/fquqfG+ywO75cjNrCs040s6r6FUrnyiyfjOzevGIvANpJ+nDgO1T0V3AeRExr8h2zcxq1RItje5CtxU9tXIO0B84Oz3fL5V9s+B2zcxq0oy33udVdCDfKiKGVTz/k6SpBbdpZlazZrz1Pq+ilx+2SFqv7Um6Gai8f7+YWZ9V5qRZRY/Ijwb+LGkG2ebLH8e5VsysCXnVSgci4k5JGwAbpaK/RcQHRbZpZtYdXrXSuS2Boamt4ZKIiEt7oV0zs9ya8db7vIpefngZsB4whQVz4wE4kJtZU2nGue+8ih6RjwA+FWX+DpnZYsFz5B2bDqwOzCq4HTOzHinzeLPoQL4y8ISkh4CPLnJGhDebMLOmUuZ15EUH8hMKrt/MrC48Iu9ARPyl7bGkMRHxhyLbMzPrrjKvWin6zs5KJ/ViW2ZmNfHGEvmoF9syM6tJmadWChmRS9q6neLqzSXMzJpGPTdflrSjpL9J+nvaJa369aUkXZVef1DS0J70vaiplbOrCyLioYLaMjPrsXolzZK0BHAWsBPwKWAfSZ+qOu0Q4I2IWB/4HfDrnvS9N+fIzcyaVh3nyEcCf4+IGRHxIdmexbtWnbMrcEl6/H/AFyR1e/q5qDnydSXd0NGLedaR/2PONM+pJ5LGRcT4RvfDmot/Lupr/oczc8ccSeOAcRVF4ys+i7WAFyteewn4TFUVH50TEfMlzQVWAl6vtd9QXCB/DTi5oLoXR+MA/w9r1fxz0SApaDfN976oQP525RpyM7PFyExg7YrnQ1JZe+e8JGlJYHlgdncbLGqO/PmC6jUza3YPAxtI+oSkjwF7A9VTzTcAB6THewB/6klywUJG5BGxu6RVgcOBTVLx48DZEfFKEW32cU3zJ5w1Ff9cNKE05/0d4DZgCeDCiHhc0knApIi4AbgAuEzS34E5ZMG+21TEInhJ2wITgIuBR1LxlmS/gcZGxH11b9TMbDFVVCB/ADgsIh6tKh8OnBcR1Vdwzcysm4qaI1+uOogDRMQUYFBBbZqZLZaKCuSStGI7hYMLbLNuJIWkkyue/0DSCRXPx0l6Kh0PSdoulU+UNCXddjs3PZ4iaZsGfA0nSPpBb7drvUfSgZLObHQ/rPGKCqq/A26X9HlJg9IxCrglvdbsPgB2l7Ry9QuSxpDljdkuIjYGDgUmSFo9Ir4aEcOBbwL3RMTwdPy1V3tvZouVQgJ5Wix/IvAzsqWIz5Olsf15RJxXRJt1Np9sRcB/tvPafwFHR8TrABExmexW28PzVi5plKS7JP1fGtVf0XZ7rqQvSHpU0jRJF0paKpU/L+k3qfwhSeun8qGS/iTpMUl3SlonR/s/TQl97pV0ZdvIXdJ6km6V9IikeyRtnMovlnS6pL9KmiFpj7xf6+IsfTZPSjpf0uOSbpe0jKThkh5In9nEtr9e08/EaemvuOmSRqbywZKuS+c/IGmzHG0fIunp9LNyftvIXdIqkq6V9HA6tk3lJ6Sft7vSZ/zdIr83Vl+FTXNExB8iYvuIWCkd20fEjUW1V4CzgLGSlq8q34QFK3HaTGLBMsu8NgeOJEuqsy6wraSlyVb67BURm5ItDz2s4j1zU/mZwKmp7AzgkojYDLgCOL2zRiVtBXwNGEaW1GdExcvjgSMiYkvgByyc/GwNYDtgDPCrGr/WxdkGwFkRsQnwJtn3/lLgv9JnNg04vuL8ZdNfdd8GLkxlJwKPpvN/nN7fIUlrAj8Ftga2BTauePk04HcR0fZz8D8Vr20MfIksV8jxkvrX/uVaIxSyjlzScZ28HBHxsyLaraeIeEvSpcB3gfcLaOKhiHgJQNIUYCjwNvBcRDydzmkb6bcF7Ssr/m2bovossHt6fBnwmy7a3Ra4PiL+BfxL0o2pDwOBbYBrtCB3z1IV77suIlrJ9mBdrYavc3H3XLrID9kAYD1ghYo7ny8Brqk4/0qAiLhb0nKSViD7Bfq1VP4nSStJWq6TNkcCf4mIOQCSrgE2TK99EfhUxWe8XPrsAW6KiA+ADyS9CqxGlifEmlxRt+i/207ZALLUjSuRTbmUwanAZOCiirInyNbE/6mibEuyG57aJekzQNuU0nHAW1RsRg20kO+ziA4ed0jS2kDbX0LndnJqP+DNNBpsT2V/ndAsv+rPeYUuzq/+XLv8nJWlTW37K/EGsp/ZjvQDtk6/yCvraK+vvbnxjPVAUXPkJ7cdZH+uLwMcRJbOcd0i2ixCGtFcTfYLqM1vgF9LWgk+Wht/IO3kYK+o58GKC58dZoUE/gYMbZv/BvYDKnPW7FXx7/3p8V9ZcFee49tSAAAE8UlEQVTYWOCeqrZfrGj7XOA+4CuSlk4jsTHpvLeA5yR9PX1dkjSsk75a98wF3pD0ufS83c84rYSaGxFzyT7Tsal8FPB6+rwAiIiWis/4OLJbxD8vaUVleTy+VlH/7cARbU/Sz6+VXGG/cZUtNTyK7AfwEmCLiHijqPYKdDLwnbYnEXGDpLWAv0oKsumQfSNiVk8bioh/STqIbHpjSbL/IStH0StKeoxs5LRPKjsCuEjS0WRZJw/qoo2HlaUYfgx4hWyOdm56eSxwjqRjgf5kv3in9vTrskUcAJwraVlgBgt/Zv+S9CjZ9//gVHYCcGH67N9jQY6OdkXETEm/BB4iu/37KRZ8xt8Fzkp1LQncTbbyykqsqDs7f0s2bzue7ELPO3VvZDEj6XlgRNtqmR7WNTAi3kmB5G5gXFp9Yw0k6S7gBxExqQ51tX3GSwITyfJ9TOxpvdacilq18n1gTeBY4GVJb6XjbUlvdfFeK974dIF1MnCtg3ifdEL6jKcDzwHXNbg/VqBCRuRmZtZ7mv52eTMz65wDuZlZyTmQm5mVnAO5LUJSS0W+j2vS6pbu1jVK0h/S410kHdPJuStI+nY32mg302NH5VXnXFxL7piUP2V6rX00K5IDubXn/XRzyaeBD6laZ5xuFqr5ZyciboiIzvK0rECWY8TMauBAbl25B1g/jUT/lvLPTAfWljRa0v2SJqeR+0AASTsqy+o4mQV5YBbKny1pNWWZ/6amYxuyZFzrpb8GfpvOOzpl6XtM0okVdf1EWXa/e4GNuvoiJP1Hqmeqsux/lX9lfFHSpFTfmHT+EpJ+W9H2t9qpcxNl2QWnpHM2qP3ba9ZzDuTWoXQzyU5kd39Clsnv7JTJ712y+wS+GBFbkGWAPEpZBsfzga+Q5aBZvYPqTydL7DQM2IIsV80xwLPpr4GjJY1ObY4EhgNbStpe0pZkaQmGAzsDW+X4cn4fEVul9p5k4bQLQ1MbXya743Lp9PrclCVwK+A/JH2iqs5DgdNSfpoROMGUNYiT4lh7lkk3k0A2Ir+A7AavFyLigVS+NVkK3vtSwqWPkeV/2Zgs498zAJIuB8a108a/AftDlisEmKtFd5UanY62bQMHkgX2QcDEiHgvtdFZ/po2n5b0c7Lpm4FkO5y3uTpldnxG0oz0NYwGNquYP18+tf10xfvuB34iaQjZL4pncvTDrO4cyK0971dnQUzBujKrpYA7ImKfqvPqmYRJwH9Xb0Yi6chu1HUxsFtETJV0IDCq4rX2Mg6KLDd7ZcBH0tCPToqYIOlBspH8zZK+FRGVWTHNeoWnVqy7HiDbDKNtp6IBkjYkS9A0VNJ66bx9Onj/naRNM9J89PJkCcgqN+e+DTi4Yu59LUmrkuWH2U3ZbjuDyKZxujIImKVss4SxVa99XVK/1Od1ybJQ3gYcls5H0oaSBlS+SdK6wIyIOB24Huhy5x6zInhEbt0SEa+lke2VStvRAcdGxNOSxgE3SXqPbGpmUDtVfI8s58shZLmvD4uI+yXdl5b33ZLmyT8J3J/+IniHLNPkZElXkWVmfJUsS2RXfgo8SJYh8sGqPv2DLFPgcsChKQvl/5DNnU9W1vhrwG5Vde4J7CdpHvBP4Jc5+mFWd861YmZWcp5aMTMrOQdyM7OScyA3Mys5B3Izs5JzIDczKzkHcjOzknMgNzMruf8H45APxUjCAAEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"nopol\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=1)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "\n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['NOT-nopol-gen','nopol-gen']); ax.yaxis.set_ticklabels(['nopol-gen' , 'NOT-nopol-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "    auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "    print('ROC AUC: %f' % auc)\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: -0.111111\n",
      "ROC AUC: 0.928571\n",
      "Accuracy: 0.733333\n",
      "Precision: 0.000000\n",
      "Recall: 0.000000\n",
      "F1 score: 0.000000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEkCAYAAAACf6vCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcXFWd/vHPk4BAICxhXwKBsIMQAiKLIyDKAEYBZZNFVMYMiwgiIA6bOCwqwzoCGlB2kSCCLLIJP0QQkCQEJCAgAYQkLGHfJkv39/fHPU0qTafrVnXdqlvp553XfXXdW7fOOd1dOX3qLN+jiMDMzMpnQKsLYGZmPXMFbWZWUq6gzcxKyhW0mVlJuYI2MyspV9BmZiXlCtr6TNIikm6S9Laka/uQzr6S7mhk2VpB0q2SDmh1Oaz9uYLuRyTtI2mcpPckTUsVyWcakPTuwPLA0hGxR72JRMRVEbFDA8ozF0nbSgpJ13e7vnG6fk/OdH4k6cpq90XEThFxWZ3FNfuIK+h+QtKRwDnAaWSV6arABcAuDUh+NeDpiJjdgLSK8hqwpaSlK64dADzdqAyU8f8paxi/mfoBSUsAPwYOjYjfR8T7ETErIm6KiKPTPQtJOkfS1HScI2mh9Ny2kl6S9H1Jr6bW9zfTcycDJwJ7pZb5gd1bmpKGpZbqAun8G5ImS3pX0nOS9q24fl/F67aS9HDqOnlY0lYVz90j6b8l3Z/SuUPSMr38GGYCNwB7p9cPBPYCrur2szpX0ouS3pE0XtK/pes7Av9V8X0+WlGOUyXdD3wArJGu/Ud6/kJJ11Wk/1NJd0lS7l+g9VuuoPuHLYGFget7uec4YAtgBLAxsDlwfMXzKwBLACsDBwLnS1oqIk4ia5VfExGLRcSveiuIpEWB84CdImIwsBUwsYf7hgC3pHuXBs4CbunWAt4H+CawHPAJ4Kje8gYuB76eHv878Dgwtds9D5P9DIYAvwGulbRwRNzW7fvcuOI1+wOjgcHAC93S+z7wyfTH59/IfnYHhGMsWA6uoPuHpYHpVbog9gV+HBGvRsRrwMlkFU+XWen5WRHxR+A9YJ06y9MJbChpkYiYFhGTerjni8AzEXFFRMyOiKuBfwBfqrjnkoh4OiI+BMaSVazzFBF/BYZIWoesor68h3uujIjXU55nAgtR/fu8NCImpdfM6pbeB2Q/x7OAK4HDIuKlKumZAa6g+4vXgWW6uhjmYSXmbv29kK59lEa3Cv4DYLFaCxIR75N1LRwETJN0i6R1c5Snq0wrV5y/XEd5rgC+A2xHD58oJB0l6cnUrfIW2aeG3rpOAF7s7cmIeAiYDIjsD4lZLq6g+4cHgBnArr3cM5VssK/Lqnz8439e7wODKs5XqHwyIm6PiC8AK5K1ii/KUZ6uMk2ps0xdrgAOAf6YWrcfSV0QxwB7AktFxJLA22QVK8C8uiV67a6QdChZS3xqSt8sF1fQ/UBEvE02kHe+pF0lDZK0oKSdJP0s3XY1cLykZdNg24lkH8nrMRH4rKRV0wDlD7uekLS8pF1SX/QMsq6Szh7S+COwdpoauICkvYD1gZvrLBMAEfEcsA1Zn3t3g4HZZDM+FpB0IrB4xfOvAMNqmakhaW3gFGA/sq6OYyT12hVj1sUVdD+R+lOPJBv4e43sY/l3yGY2QFaJjAMeA/4OTEjX6snrTuCalNZ45q5UB6RyTAXeIKssD+4hjdeBUWSDbK+TtTxHRcT0esrULe37IqKnTwe3A7eRTb17Afg/5u6+6FqE87qkCdXySV1KVwI/jYhHI+IZspkgV3TNkDHrjTyYbGZWTm5Bm5mVlCtoM7OScgVtZlZSrqDNzEqqt4ULLTVr+mSPXtrHHL7Zsa0ugpXQBc+P7XNsk1rqnAWXWaMpsVTcgjYzK6nStqDNzJqqY1b1e5rMFbSZGUBnTwtaW8sVtJkZEOEK2sysnNyCNjMrKbegzcxKqrOj1SX4GFfQZmYAHeXb89gVtJkZHiQ0MysvDxKamZWUW9BmZiXlQUIzs5LyIKGZWUm5i8PMrKQ8SGhmVk4R5euDdjxoMzPIujjyHlVI+rWkVyU9XnFtiKQ7JT2Tvi5VLR1X0GZmkHVx5D2quxTYsdu1Y4G7ImIt4K503itX0GZmkAXsz3tUERH3Am90u7wLcFl6fBmwa7V0XEGbmUFNXRySRksaV3GMzpHD8hExLT1+GVi+2gs8SGhmBjXN4oiIMcCYerOKiJBUdZNaV9BmZtCMedCvSFoxIqZJWhF4tdoL3MVhZgaNHiTsyY3AAenxAcAfqr3ALWgzM2joQhVJVwPbAstIegk4CfgJMFbSgcALwJ7V0nEFbWYGRI7ZGbnTivjaPJ7avpZ0XEGbmYFjcZiZlZZjcZiZlZRb0GZmJeUWtJlZSTlgv5lZSbkFbWZWUu6DNjMrKbegzcxKyi1oM7OScgvazKykPIvDzKyk3II2MyupqBo/v+lcQZuZgVvQZmal5QrazKykPEhoZlZS7oM2Myspd3GYmZWUK2gzs5LyUm8zs3KKTvdBm5mVk2dxmJmVlFvQZmYl5UFCM7OScgVt1Rx/2lnce//fGLLUktxw5S8AuP3uv3DBr65k8gsvcvVF57Dhemu3uJTWSgsstCBHXnMyCyy0AAMGDuSRWx/klrOvbXWx2l8JF6oMaHUBbG677vwFfnHWKXNdW3ON1TjntBPYdMSGLSqVlcnsGbM4d5+TOW2nYzht52NYf5sRDNtkrVYXq/11duY/msQt6JLZbMQnmTLtlbmuDR+2aotKY2U144MZAAxcYCADFxhYytZf2+noaHUJPqbQClrS2sDRwGqVeUXE54rM12x+pwHi2Jt/yrKrrcC9V9zO8xP/2eoitb8SzuIouovjWmACcDxZRd119EjSaEnjJI27+PKrCy6aWfuKzuD0nY/huC0PYtjGw1lx7aGtLlLbi87O3EezFN3FMTsiLsx7c0SMAcYAzJo+uXx/zsxK5sN3PuCpByaxwTYjmPb0i60uTnvrhy3omyQdImlFSUO6joLzNJuvLTZkMIssPgiABRdakPU+sxEvPzulxaWaD0Rn/qMKSd+TNEnS45KulrRwPUUqugV9QPpa2a0RwBoF59u2jj7pJzz8yGO89dY7bL/rfhxy4P4ssfhinH72hbzx1tsccvRJrLvWGow5+9RWF9VaZInlluLrZx7KgAED0AAx/pYHePzuCa0uVvub3ZhBQkkrA98F1o+IDyWNBfYGLq01rUIr6IhYvcj050dnnHxsj9c/v83WTS6JldWUf/yL07/4g1YXY/7T2C6OBYBFJM0CBgFT60mk0C4OSYMkHS9pTDpfS9KoIvM0M6tLDV0clRMa0jH6o2QipgD/A/wLmAa8HRF31FOkovugLwFmAlul8ynAKfO+3cysRToj9xERYyJis4pjTFcykpYCdgFWB1YCFpW0Xz1FKrqCHh4RPwNmAUTEB4AKztPMrGYNnGb3eeC5iHgtImYBv2dOI7UmRQ8SzpS0CNnAIJKGAzMKztPMrHaN64P+F7CFpEHAh8D2wLh6Eiq6gj4JuA0YKukqYGvgGwXnaWZWuwYt9Y6IhyT9jmyR3mzgEdL6jloVPYvjTkkTgC3IujYOj4jpReZpZlaXBs7iiIiTyBqofVJ0LI6R6eG09HVVSUsAL0RE+faXMbN+qz/uSXgBMBJ4jKwFvSEwCVhC0sH1Tj0xM2u4ElbQRc/imApskqahbApsAkwGvgD8rOC8zczy64fxoNeOiEldJxHxhKR1I2Ky5Nl2ZlYiJWxBF11BT5J0IfDbdL4X8ISkhUhzo83MyiA6+t+ehN8ADgGOSOf3A0eRVc7bFZy3mVl+/a0FHREfAmcCZ0oaGRGVIbfeKzJvM7OalLCCbuamsRc3MS8zs5pEZ+Q+mqWZm8Z6VNDMyquELehmVtAnNzEvM7OaxOzyVdBN6+KIiBskrS3pomblaWaWWw3hRpulkApa0kaS7kj7cZ2S9iS8DrgbeKKIPM3M+qSzhqNJimpBXwT8Bvgq8BowEXgWWDMizi4oTzOzuvWnQcKFIuLS9PgpSYdHxDEF5WVm1nflW6dSWAW9sKRNmDNzY0blebf50GZmLdefotlNA86qOH+54jyAzxWUr5lZXcoYALmQCjoivIzbzNpLCbs4Cp1mJ2mQpOMljUnna0kaVWSeZmb1iM78R7NUraAlfUXS4PT4WEljJY3Imf4lwEzm7Gg7BTilrpKamRWpTafZ/Sgi3pW0FbAzcBXwi5zpD4+In5FCi0bEB3jJt5mVUFu2oIGurW5HAb+MiD8AC+VMf6akRcgGBpE0HJhRcynNzApWxgo6zyDhNEnnAzsCm0n6BPn7rk8CbgOGSroK2JosRrSZWalER/k+3OepoPck69r434h4U9JKwLF5Eo+IOyVNALYg69o4PCKm111aM7OCNLNlnNc8K2hJi1ec3lZx7T2ynVGqkrQbcHdE3JLOl5S0a0TcUH+RzcwaLzrbqwU9iazvuLLUXecBrJoj/ZMi4vqPXhzxlqSTAFfQZlYqbdWCjoihDUi/p77qZsagNjPLJaJ8Lehcg32S9pb0X+nxKpI2zZn+OElnSRqejrOA8fUW1sysKJ2zlftoljwLVX5OtgP3/unSB+SfB30Y2UKVa9IxAzi09mKamRUrIv/RLHm6G7aKiJGSHgGIiDfSVLuqIuJ9cs74MDNrpXYbJOwyS9IA5iw2WZqcix0lrQ0cBQyrzCsiHM3OzEqlXSvo84HrgGUlnUw2LzrvBrDXknWHXMycFYlmZqXTyK4LSUuS1XsbkjVuvxURD9SaTtUKOiIulzQe+Hy6tEdEPJ4z/dkRcWGthTIza7YGt6DPBW6LiN1Tl/CgehLJO+VtIFnAo6C2EKU3SToEuJ6KGBwR8UYNaZiZFa6zQUu9JS0BfJYU1iIiZpJNlqhZnlkcxwFXAysBqwC/kfTDnOkfABwN/JVset14YFw9BTUzK1JnKPchabSkcRXH6IqkVifbLPsSSY9IuljSovWUKU8L+uvAJilUKJJOBR4BTq/2wohYvZ5CmZk1Wy0LVSJiDDBmHk8vAIwEDouIhySdSzab7YRay5Snu2Iac1fkC6RrVXlHFTNrF9Gp3EcVLwEvRcRD6fx3ZBV2zXoLlnQ2WZ/zG8AkSben8x2Ah3OmfwlZt0bljirXAjfXU1gzs6I0ahZHRLws6UVJ60TEU8D2wBP1pNVbF0fXTI1JwC0V1x+sIf3hEbGXpK9BtqOKpPJNNjSzfq+jo6FbtB4GXJVmcEwGvllPIr0FS/pVnQWr5B1VzKwtNDJYUkRMBDbrazpVBwlTpXoqsD6wcEUB1s6RvndUMbO20MwYG3nlmcVxKdlO3P8D7ETWVM/1rXhHFTNrF51tGm50UETcDhARz0bE8WQVdVWSPgtsALwLvAOsn66ZmZVKhHIfzZKnBT0jBUt6VtJBZDMxBudM/+iKxwsDm5PN6qgaLGnddXfPmYX1Jy+880qri2AldEED0mjXLo7vAYsC3yXri14C+FaexCPiS5XnkoYC59RYRjOzwnV0NnQWR0PkCZbUNdn6XeYE7a/XS8B6fUzDzKzhytgH3dtClevpZTAwIr5SLXFJ/1uRxgBgBDChxjKamRWuhD0cvbagf96A9CsDI80Gro6I+xuQrplZQ7VVCzoi7upr4hFxWV/TMDNrhjLu6p03HnRdJP2dnj85CIiI2KjI/M3M8sq1j1+TFVpBA7emr1ekr/umr95lxcxKpaOdW9CSFoqIWuNofCEiNqk4P1bShIjwTt9mViqdlK+CzrOjyuapq+KZdL5xmp2RhyRtXXGyVZ48zcyaLVDuo1nytKDPA0YBNwBExKOStsuZ/oHAr9MeXQBvkXORi5lZM7VrH/SAiHihWxjnjjyJR8R4YOOuCjoi3q69iGZmxWtmyzivPN0NL0raHAhJAyUdATydJ3FJy0v6FfDbiHhb0vqSDuxLgc3MijC7hqNZ8lTQBwNHAqsCr5CFDj04Z/qXAreT7QgOWcV+RG1FNDMrXlv2QUfEq8Dedaa/TESMlfTDlNZsSbm6R8zMmqn6XrDNl2dHlYvoYbFJRIzOkf77kpZmzpZXWwDuhzaz0injNLs8g4R/qni8MLAb8GLO9I8EbgSGS7ofWBZwoGczK512C5YEQERcU3ku6Qrgvt5eI+lTwIsRMUHSNsB/Al8F7iALOWpmViplnGZXz6KR1YHlq9zzS2BmerwVcBxwPvAmMKaOPM3MCtUh5T6aJU8f9JvMHdP5DaDaUu2BEfFGerwXMCYirgOukzSx3sKamRWljC3oXitoZatTNibbhxCgMyLXzl0DJS0QEbOB7YHKAcWiAzSZmdWs7WZxRERI+mNEbFhjulcDf5Y0HfgQ+AuApDXxLA4zK6F2ncUxUdImEfFI3kQj4lRJdwErAndUtLoHAIfVUU4zs0K11SyOii6KTYCHJT0LvM+cYPsje0s4Ih7s4VquJeJmZs3Wbl0cfwNGAl9uUlnMzFqmjEuce6ugBRARzzapLGZmLdNuLehlJR05rycj4qwCymNm1hLtNs1uILAYlHBo08yswdqtgp4WET9uWknMzFqo0XvGShoIjAOmRMSoetKo2gdtZtYfFBCI/3DgSWDxehPoLRbH9vUmambWbqKGoxpJqwBfBC7uS5nmWUFXxNIwM5vvdSr/IWm0pHEVR/f4+OcAx9DHrm3HxTAzo7aaNCLGMI/InJJGAa9GxHhJ2/alTK6gzcxo6CyOrYEvS9qZbJOTxSVdGRH71ZpQPfGgzczmO43qg46IH0bEKhExjGw/17vrqZzBLWgzMwBml3DemitoMzOKiWYXEfcA99T7elfQZmZAZwkDjrqCNjOj/ZZ6m5n1G+VrP7uCNjMD3II2Myut2SpfG9oVtJkZ7uIwMystd3GYmZWUp9mZmZVU+apnV9BmZgDMLmEV7QrazAy3oM3MSsuDhGZmJRUlbEO7gjYzo5wtaAfsL7GfnHsSf3vyT9z6l7GtLoqVzL/vsC2THr+XfzxxH8ccfWirizNf6CRyH83iCrrErvvtTXxzr++0uhhWMgMGDOC8c09l1Jf245Mbb8dee+3Keuut1epitb0OIvfRLK6gS+zhBybw1ptvt7oYVjKbf2oTnn32eZ577l/MmjWLsWP/wJe/9O+tLlbb66zhaBZX0GZtZqWVV+DFl6Z+dP7SlGmstNIKLSzR/CFq+NcshQ4SSloW+DYwrDKviPjWPO4fDYwGWGbRoSy+8DJFFs/M7CNlHCQsehbHH4C/AH8COqrdHBFjgDEAw5cZWb45L2YlMHXKywxdZaWPzldZeUWmTn25hSWaP/THaXaDIuIHBedh1q88PG4ia665OsOGDWXKlJfZc89d2P/rnsnRV7OjfBV00X3QN0vaueA85lvnjDmN3912KauvuRr3PXYre+y7S6uLZCXQ0dHB4Ucczx9v+Q2PP3YPv/vdTTzxxNOtLlbbixqOZlEU+FdD0rvAosDMdAiIiFi82mvdxWE9eeGdV1pdBCuh2TOnqK9p7LPabrnrnN+8cH2f88uj0C6OiBhcZPpmZo1Sxj7oQrs4lNlP0gnpfKikzYvM08ysHv1xHvQFwJbAPun8PeD8gvM0M6tZGZd6Fz2L49MRMVLSIwAR8aakTxScp5lZzZq5hDuvoivoWZIGkgY+08KVMs4HN7N+rsgJE/UquoI+D7geWE7SqcDuwPEF52lmVrN+t2lsRFwlaTywPdkUu10j4ski8zQzq0cZP9oXHYtjCPAqcHXFtQUjYlaR+ZqZ1apR0+wkDQUuB5Yn694dExHn1pNW0V0cE4ChwJtkLeglgZclvQJ8OyLGF5y/mVkuDezimA18PyImSBoMjJd0Z0Q8UWtCRU+zuxPYOSKWiYilgZ2Am4FDyKbgmZmVQkdE7qM3ETEtIiakx+8CTwIr11OmoivoLSLi9q6TiLgD2DIiHgQWKjhvM7PcaokHLWm0pHEVx+ie0pQ0DNgEeKieMhXdxTFN0g+A36bzvYBX0tS7MvbJm1k/VUsXR2Vo5HmRtBhwHXBERLxTT5mKbkHvA6wC3JCOoenaQGDPgvM2M8stInIf1UhakKxyvioifl9vmYqeZjcdOAxA0ooRMa3i6X8WmbeZWS0aNUgoScCvgCcj4qy+pNXMPQlvaWJeZmY16YjO3EcVWwP7A5+TNDEddcXFL7oPulJT4qeamdWjUZPsIuI+GlTfNbOCvqiJeZmZ1aSMS72b1sURERdIWlLScc3K08wsrzKGGy2kgk6B+cdIulnSf0haVNKZwNPAckXkaWbWF42cxdEoRXVxXA78mWyayY7AOGAisFFEeH94MyudMnZxFFVBD4mIH6XHt0vaA9g3ovrwp5lZK3SWsHoqbJBQ0lLMGcl8HVgizQ8kIt4oKl8zs3r0pxb0EsB45p5qMiF9DWCNgvI1M6tLv9lRJSKGFZGumVlRytiCLnSanTL7SToxna8qafMi8zQzq0ct0eyapeh50BcAWwJfS+fvAucXnKeZWc06I3IfzVL0SsJPR8RISY8ARMSbkj5RcJ5mZjXLEWOj6YquoGel2M8BIGlZHAfazEqomV0XeRVdQZ8HXA8sJ+lUYHfg+ILzNDOrWTO7LvIqOh70VZLGA9uTTbnbNSKeLDJPM7N69LsWtKQtgEkRcX46X1zSpyOirv25zMyKUsYWdNGzOC4E3qs4fy9dMzMrlc7oyH00S9F90IqK5TkR0SmpmTGozcxy6XcLVYDJkr4racF0HA5MLjhPM7OalTHcaNEV9EHAVsAU4CXg08DogvM0M6tZGQP2Fz2L41Vg7yLzMDNrhH4TLKlLWpjybWBYZV4R8a0i8zUzq1UZZ3EUPWD3B+AvwJ+A5g19mpnVqF8F7E8GRcQPCs7DzKzP+uMsjpsl7VxwHmZmfVbGWRxFt6APB/5L0kxgJtly74iIxQvO18ysJv2uDzoiBheZvplZo5RxFkezdlQ5IZ0P9Y4qZlZGZZwH3awdVfZJ5+/hHVXMrIQ6OjtzH83iHVXMzOiH4Ubxjipm1ib63SAh3lHFzNpEGQcJvaOKmRmN7eKQtCNwLjAQuDgiflJPOkXH4lgV+AC4qfJaRPyryHzNzGrV2aDBv9Stez7wBbIong9LujEinqg1raK7OG4h638WsDCwOvAUsEHB+ZqZ1aSBHRybA/+MiMkAkn4L7AKUq4KOiE9WnksaCRyS57XPTp+gQgrVhiSNjogxrS6HlYvfF401e+aU3HWOpNHMHdt+TMXvYmXgxYrnumLh16zoedBziYgJ1FnQfs6bHFhP/L5okYgYExGbVRyF/KEsug/6yIrTAcBIYGqReZqZtdgUYGjF+SrpWs2KbkEPrjgWIuuT3qXgPM3MWulhYC1Jq6eFeXsDN9aTUNF90CcXmX4/4n5G64nfFyUUEbMlfQe4nWya3a8jYlI9aanIydmSbqKXwdGI+HJhmZuZtbmip9lNBlYArkznXwNeAW4oOF8zs7ZXdAt6XERsVu2amZl9XNGDhItKWqPrRNLqwKIF51k3ScdJmiTpMUkTJbXVlEBJ90h6StKjkh6WNKLVZSobSSHpzIrzoyT9qOJ8tKR/pONvkj6Trl+f3hP/lPR2ejxR0lbd0h8g6TxJj0v6e/o9rN60b7BBJD2fyv+YpD9LWq3VZeqPiu7i+B5wj6TJZKsJVwP+s+A86yJpS2AUMDIiZkhaBmjH0Kj7RsQ4Sd8EziBbbmpzzAC+Iun0iJhe+YSkUWTvz89ExPS0sOoGSZtHxG7pnm2BoyJi1DzS3wtYCdgoIjolrQK8X9Q3U7Dt0s/hZLIgZ99udYH6m0Jb0BFxG7AW2d6E3wXWiYjbi8yzD1YEpkfEDICImB4RUyVtmloQ4yXdLmlFgHT90XScIenxdP0bkn7elaikm9N/aiTtIOkBSRMkXStpsXT9eUknp+t/l7Ruur6YpEsqWjJf7S2dbh4gW9H0MZIGSRor6YnUMnxI0mb1lLENzSab/fC9Hp77AXB0V8WdFlZdBhxaQ/orAtMiojOl8VJEvAm9/mx3TC32Can1fXO6/iNJR3UlnFrlw9Lj/VILf6KkXyqL/4Ck9ySdmt6XD0paPl1fPv2uu96zW/WWTje9vZeWlXSnsk+eF0t6ITVuai6jfVzRW14NAo4GvhMRjwKrplZKGd0BDJX0tKQLJG0jaUHgf4HdI2JT4NfAqen+S4DDImLjPImnN+3xwOcjYiQwDqhcyDM9Xb8Q6PpPeQLwdkR8MiI2Au7OkU6XHZn3YOwhwJsRsX7KY9M+lLEdnQ/sK2mJbtc3AMZ3uzaO2mLHjAW+lCqlMyVtAvP+2UpaGLgI+BLZ72GFahlIWo+spb51RIwAOoB909OLAg+m9+W9zGn1ngf8OV0fCUyqkk6l3t5LJwF3R8QGwO+AVftQRuum6C6OS8je8Fum8ynAtcDNBedbs4h4T9KmwL8B2wHXAKcAGwJ3SoJsTuM0SUsCS0bEvenlVwA7VcliC2B94P6U1ifIWiZdfp++jge+kh5/nmySe1cZ30x/4HpL5yplk+MXA+bVB/0ZslCIRMTjkh7rQxnbTkS8I+lysk91HzY47ZckrQN8Lh13SdoDWISef7brAs9FxDMAkq6k+hLu7ckq84dTWosAr6bnZjLn/9d45nRxfQ74eipjB/C2pP17SQfg/0kaQrZV3QnzKMtngN1SurdJerMPZbRuiq6gh0fEXpK+BhARHyj9tsoovXHvIes3/zvZR9tJEbFl5X2pgp6X2cz9yWThrpcBd0bE1+bxuhnpawe9/16qpbMv2Zv+DLLW/1ck7UbW0gH4jz6knbeM7eAcYAJZI6LLE2SVyt0V1zYF5rnIQNlA8i/T6YkRcWPqJrsVuFXSK8CuZJ/QPvazVe8Dub29ly6LiB/28JpZMWdqVp730rzSgayh8hZwFXAyWYv/UOa0eHeuM+1aytivFT2LY6akRZiz5dVw5vwnLxVJ60haq+LSCOBJYFllA4hIWlDSBhHxFvCW0gg/c38sfB4YoWw0fyhZ6EGAB4GtJa2Z0lpU0tpVinUnFf2fkpbKk056858AbCFp3Yi4PiJGpGMccD+wZ3r9+kBX1MF6ytiWIuINsu4g217JAAAGi0lEQVSIAysu/wz4qaSl4aPK8xtkmx/PK52HKn62N0oaKWml9PoBwEbAC8z7Z/sPYFj6vwHZWoEuz5N1R3RFguyaDXIXsLuk5dJzQ1R9lsVdwMHp/oGpe6dqOhExGzgC+LqkIRFxfsX3O5W530s7AEv1oYzWTdEV9EnAbWR9u1eR/dKOKTjPei0GXKZs4Owxso+jJ5Jt0/VTSY8CE4GuaVXfBM6XNJGstdDlfuA5stbYeWStNCLiNbL/7Fen9Ls+3vbmFGApZYNDj5KNqudKJyI+BM4kGwPo7gKyPzxPpDwmkfV111PGdnYmsEzXSUTcSDbO8FdJ/yDrG94vIqbVkOZywE3KBo0fI2sF/3xeP9uI+D+yLo1bJE1g7i6G64AhkiYB3wGeTuV8gqw/+46U1p1kg5O9ORzYLn0yHA+snzed9P1fTc+DpScDO6Tvdw/gZeDdOsto3RS6UAUgtUa2IKvEHuw+tWl+oGxk/eaI2LDFRckljaYvGBH/l1pufyKbYTOzxUXr91R9Gl+pSFoI6EjxJ7YELkyDgtYAhfT9pI8yb0XE2xHxuqQPyPrh1pb0c1cELTeIbABoQbI/nIf4d2J1WhUYm7pzZuIZGQ1VSAta0kPAbmke8QiyFtrpZP1xsyKit4EqMzOjuNHTRdIAAsB+ZOH2zkx/ZScWlKeZ2XylqEHCykGzz5ENDtK1usrMzKorqgV9t6SxwDSyaTd3AyhbJu2+TjOzHIpqQR9BtursebLAM7PS9RWA4wrK0xpEUkdaqvy4spgRg/qQ1raaE1viy5KO7eXeJSXl2vW92+vmillR7Xq3ey6VtHsNeQ1LU8rMCldIBR2Z30bE2RExBbJIYRHxSImDJdkcH6aFCBuSfeI5qPJJZWp+76RVdj/p5ZYlyeKEmBnFL1Sp9OMm5mWN8xdgzdRyfEpZDIvHyRYfVY3ORkXMDlVE+lPP0dV+AgxPrfcz0n1HK4up/JiysJddaR2nLLDVfcA61b4JSd9O6Twq6bpunwo+L2lcSm9Uun+gsiiFXXl/LEyupA00J1rbY5p7JapZnzWzgi5tDA7rmaQFyIJA/T1dWgu4IEUue5++RWf7WHQ14Fjg2dR6PzotHV6LbLn8CGBTSZ9VFtRq73RtZ+BTOb6d30fEp1J+TzL3Eu9hKY8vAr9I38OBZKsrP5XS/7Y+Hnj/IODctDBjM+ClHOUwy62ohSpbRMSD3S6XMlC/9WiRtIQdshb0r8iC0L9Q8XudV+S7vNHZeoqutlS3e3ZIxyPpfDGyCnswcH1EfJDyyLOl/YaSTiHrRlmMbMflLmPTDKNnlG0usW7Kd6OK/uklUt5PV7zuAeA4ZUH5f9/1PZs1SlGzOC4gBXnpEhF/Kygva7wPuy/XTZVw5c4gPUa+U2O32RJwekT8cq6L0hF1pHUpsGtEPCrpG8C2Fc91X60VKe/Duo+ZpGX92U0Rv0mLsr4I/FHSf0ZEZSQ8sz5pZheHzV/qic5Wqafoau+StY673A58q6Jve2Vl0dHuBXaVtIikwWTdKdUMJovlvSAfD0q/h7Log8OBNYCnUt4Hp/uRtLakufbTVLbf5uSIOA/4A9lKWbOGKaoFvUZvHzsj4ssF5WtNEhGvpZbo1SlgDsDxEfG0pK7obB+QdZEM7iGJw4Exkg4kiwl8cEQ8IOn+NI3t1tQPvR7wQGrBv0cWXW6CpGuAR8mivz2co8gnAA8Br6WvlWX6F/A3YHHgoBRE6mKyvukJyjJ/jSyeTKU9gf0lzSKL4nZajnKY5VZULI5n6CUwfET8ueGZmpnNZ4pqQb/rStjMrG+K6oN+vqB0zcz6jcIC9qfBnEOZsyPyJLI5tK8UkqGZ2XymkBa0pK2ZM3BzeToAHkrPmZlZFUUNEj5INir/SLfrI4BfRsSnG56pmdl8pqg+6MW7V84AETGRnqdcmZlZN4UF7O9h2S6ShhSYp5nZfKWoyvJssu3Wt5E0OB3bArem58zMrIoiZ3GMAo5h7lkcZ0TETYVkaGY2nymsgjYzs74pKtzoib08HRHx30Xka2Y2Pylqmt33e7i8KFkQ9KUjYrGGZ2pmNp8pvIsjhYM8nKxyHgucGRGvFpqpmdl8oKhgSV1T6o4ki717GTAyIt4sKj8zs/lNUX3QZ5BtFjoG+GREvFdEPmZm87Oi+qA7gRnAbObeTkhkg4SLNzxTM7P5jKfZmZmVlJddm5mVlCtoM7OScgVtZlZSrqDNzErq/wMfnG43FY62aQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"SequenceR\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=1)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "    \n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['SequenceR-gen','NOT-SequenceR-gen']); ax.yaxis.set_ticklabels(['SequenceR-gen' , 'NOT-SequenceR-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "    auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "    print('ROC AUC: %f' % auc)\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: -0.111111\n",
      "ROC AUC: 1.000000\n",
      "Accuracy: 0.733333\n",
      "Precision: 0.000000\n",
      "Recall: 0.000000\n",
      "F1 score: 0.000000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEWCAYAAABLzQ1kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XecXGW9x/HPNwlSE7p0CL1KCUWaNBERQxFpl45opClIu3hBIUix3CCgIEaQDhKa0osgUsSQEAIkVCkBktAh1BuS3d/94zwbhs2Wmdk5M2d3vm9e57UzZ848z7PZ5bfP/M5zfkcRgZmZFU+/Rg/AzMw65gBtZlZQDtBmZgXlAG1mVlAO0GZmBeUAbWZWUA7Q1mOS5pZ0s6Rpkq7tQTv7SLqrlmNrBEm3Szqg0eOw3s8BuolI2lvSWEkfSZqaAsnmNWh6N2AxYOGI2L3aRiLiyojYrgbj+QJJW0kKSTe2279O2n9fme2cIumK7o6LiG9FxKVVDtdsFgfoJiHpaOBs4AyyYLoscD6wcw2aXw54LiJm1qCtvLwFbCJp4ZJ9BwDP1aoDZfz/lNWMf5magKT5gVOBwyPihoj4OCJmRMTNEXFcOmZOSWdLmpK2syXNmV7bStJrko6R9GaafR+UXhsO/BzYM83MD24/05Q0OM1UB6TnB0p6UdKHkl6StE/J/gdL3reppDEpdTJG0qYlr90n6ReSHkrt3CVpkS7+GT4D/grsld7fH9gTuLLdv9U5kl6V9IGkRyV9Le3fHvifku/z8ZJxnC7pIeATYIW07/vp9T9Iur6k/V9JukeSyv4BWtNygG4OmwBzATd2ccyJwMbAusA6wEbASSWvLw7MDywFHAycJ2nBiDiZbFZ+TUTMFxEXdTUQSfMC5wLfioiBwKbA+A6OWwi4NR27MHAWcGu7GfDewEHAl4EvAcd21TdwGbB/evxNYAIwpd0xY8j+DRYCrgKulTRXRNzR7vtcp+Q9+wHDgIHApHbtHQN8Jf3x+RrZv90B4RoLVgYH6OawMPB2NymIfYBTI+LNiHgLGE4WeNrMSK/PiIjbgI+AVascTyuwlqS5I2JqREzs4JhvA89HxOURMTMirgaeAXYsOebiiHguIj4FRpEF1k5FxL+AhSStShaoL+vgmCsi4p3U5whgTrr/Pi+JiInpPTPatfcJ2b/jWcAVwI8i4rVu2jMDHKCbxTvAIm0phk4syRdnf5PSvllttAvwnwDzVTqQiPiYLLVwCDBV0q2SVitjPG1jWqrk+etVjOdy4Ahgazr4RCHpWElPp7TK+2SfGrpKnQC82tWLETEaeBEQ2R8Ss7I4QDeHh4HpwC5dHDOF7GRfm2WZ/eN/uT4G5il5vnjpixFxZ0R8A1iCbFb8pzLG0zamyVWOqc3lwGHAbWl2O0tKQRwP7AEsGBELANPIAitAZ2mJLtMVkg4nm4lPSe2blcUBuglExDSyE3nnSdpF0jyS5pD0LUm/ToddDZwkadF0su3nZB/JqzEe2ELSsukE5U/bXpC0mKSdUy56OlmqpLWDNm4DVklLAwdI2hNYA7ilyjEBEBEvAVuS5dzbGwjMJFvxMUDSz4FBJa+/AQyuZKWGpFWA04B9yVIdx0vqMhVj1sYBukmkfOrRZCf+3iL7WH4E2coGyILIWOAJ4ElgXNpXTV93A9ekth7li0G1XxrHFOBdsmB5aAdtvAMMJTvJ9g7ZzHNoRLxdzZjatf1gRHT06eBO4A6ypXeTgP/ji+mLtotw3pE0rrt+UkrpCuBXEfF4RDxPthLk8rYVMmZdkU8mm5kVk2fQZmYF5QBtZlZQDtBmZgXlAG1mVlBdXbjQUDPeftFnL202R25wQqOHYAV0/sujelzbpJKYM8ciK9Sllopn0GZmBVXYGbSZWV21tjR6BLNxgDYzA2gpXjlzB2gzMyCio4oDjeUAbWYG0OoAbWZWTJ5Bm5kVlE8SmpkVlGfQZmbFFF7FYWZWUD5JaGZWUE5xmJkVlE8SmpkVlGfQZmYF5ZOEZmYF5ZOEZmbFFFG8HLTrQZuZQZaDLnfrhqQ/S3pT0oSSfQtJulvS8+nrgt214wBtZgZZiqPcrXuXANu323cCcE9ErAzck553yQHazAxqOoOOiPuBd9vt3hm4ND2+FNilu3acgzYzA2iZUfahkoYBw0p2jYyIkd28bbGImJoevw4s1l0/DtBmZlDRKo4UjLsLyF29PyR1e5NaB2gzM6jHhSpvSFoiIqZKWgJ4s7s3OAdtZga1PknYkZuAA9LjA4C/dfcGz6DNzKCmF6pIuhrYClhE0mvAycAvgVGSDgYmAXt0144DtJkZEBWcJOy2rYj/6uSlr1fSjgO0mRm4WJKZWWG5FoeZWUF5Bm1mVlCeQZuZFZRn0GZmBTXTBfvNzIrJM2gzs4JyDtrMrKA8gzYzKyjPoM3MCsozaDOzgvIqDjOzgopu6+fXnQO0mRk4B21mVlgO0GZmBeWThGZmBdXS0ugRzMYB2swMnOIwMyssB2gzs4JyDtrMrJii1eugzcyKySkOM7OC8ioOM7OC8gzazKygHKCtOyedcRb3P/QICy24AH+94gIA7rz3Ac6/6ApenPQqV//pbNZafZUGj9IaacCcc3D0NcMZMOcA+vXvz2O3/5tbf3tto4fV+xWwWFK/Rg/AvmiXHb7BBWed9oV9K62wHGef8TPWX3etBo3KimTm9Bmcs/dwzvjW8Zyxw/GsseW6DF5v5UYPq/drbS1/qxPPoAtmg3W/wuSpb3xh34qDl23QaKyopn8yHYD+A/rTf0D/Qs7+ep1mW2YnaTPgFGC51JeAiIgV8uzXrK9TP3HCLb9i0eUW5/7L7+Tl8f9p9JB6vwKu4sg7xXERcBawObAhsEH62iFJwySNlTT2wsuuznloZr1XtAZn7nA8J25yCIPXWZElVlmm0UPq9aK1teytXvJOcUyLiNvLPTgiRgIjAWa8/WLxPm+YFcynH3zCsw9PZM0t12Xqc682eji9WwFTHHnPoP8h6TeSNpE0pG3LuU+zPm2+hQYy96B5AJhjzjlYffO1ef2FyQ0eVR8QreVv3ZD0E0kTJU2QdLWkuaoZUt4z6K+mrxuU7Atgm5z77bWOO/mXjHnsCd5//wO+vsu+HHbwfsw/aD7O/O0fePf9aRx23MmstvIKjPzt6Y0eqjXI/F9ekP1HHE6/fv1QP/HorQ8z4d5xjR5W71ejGbSkpYAfA2tExKeSRgF7AZdU2lauAToits6z/b7oN8NP6HD/tltuVueRWFFNfuYVzvz2fzd6GH3PzJqeJBwAzC1pBjAPMKWaRnJNcUhaTNJFkm5Pz9eQdHCefZqZVaWCFEfpgoa0DZvVTMRk4H+BV4CpZOfi7qpmSHnnoC8B7gSWTM+fA47KuU8zs8q1RtlbRIyMiA1KtpFtzUhaENgZWJ4s9s0rad9qhpR3gF4kIkYBrQARMRMo3mJDM2t6NVxmty3wUkS8FREzgBuATasZU94nCT+WtDDZiUEkbQxMy7lPM7PK1W6Z3SvAxpLmAT4Fvg6MraahvAP00cBNwIqSHgIWBXbLuU8zs8rVKEBHxGhJ1wHjgJnAY6TrOyqV9yqOcZK2BFYlu8z72TTlNzMrlhpe6h0RJwMn97SdvGtx7Npu1yqSpgFPRsSbefZtZlaJZrwn4cHAJsA/0vOtgEeB5SWdGhGX59y/mVl5mjBADwBWj4g3IFsXDVxGdoXh/YADtJkVQxPeUWWZtuCcvJn2vZuusDEzK4YmnEHfJ+kWoO1+PN9N++YF3s+5bzOz8jVhgD4c2JWsHjRk6Y3rIyIA1+kws8KIliZLcaRAfD1wvaShEXFLnv2ZmVWtgDPoet409tQ69mVmVpFojbK3eqnnTWNVx77MzCpTwBl0PQP0D+vYl5lZZYqXgs69HvQvJA0AiIhHJA2SdHGefZqZVSNmtpa91UveOegBwGhJa0v6BjCG7EpCM7Niaa1gq5O8V3H8VNLfgdHAe8AWEfGfPPs0M6tGEWtx5J3i2AI4l2wFx33A7yQt2eWbzMwaodlm0GT35do9Ip6CWdXt7gVWy7lfM7OKFHEGnXeA3iQiZhVZjYgbJP0z5z7NzCpXwFUcuQRoSftGxBXAkVKHy5/PyqNfM7NqxcxGj2B2ec2g501fB+bUvplZTUVvnEGnvPHdEfGhpBOAIcAZETG+s/dExB/T1+E1G6mZWZ4KGKDLWcVxSgrOmwI7AFcCF3T1Bkl3lTz+ac+GaGaWv2gtf6uXcgJ020m+ocAfI+JvwJzdvGfRkse7VzMwM7N6KmKALicHPVXSecD2wAaSvkT3gb1461XMzLoQLcWr51ZOgN6DLLXxu4h4L11ockI371lB0k1kFezaHs8SETtVNVozs5z0qpOEkgaVPL2jZN9HwEPdtLtzyeP/rXp0ZmZ1Eq29awY9kSxVUTrqtucBLNvZGyPCF6OYWa/Sq2bQEbFMTxuXNBT4BbBc6ktZ0zGoyzeamdVZRO+aQc8iaS9ghYg4Q9LSwGIRUU7Z0LPJbhr7ZLo/oZlZIRVxBt3tMjtJvye7A/d+adcndLMOusSrwAQHZzMrutYWlb3VSzkz6E0jYoikxwAi4t201K4cxwO3pQJJ09t2RoRrcZhZofS2k4RtZkjqR1rbLGlhyr8o8nSyVR9zAeUGdTOzuuutAfo84HpgUUnDydZFl1tjY8mIWKvawZmZ1UstE7GSFgAuBNYim9x+LyIerrSdbgN0RFwm6VFg27Rr94iYUGb7t0naLiLu6v5QM7PGqfEM+hzgjojYLaWE56mmkXLLjfYHZpD9JajkNlmHAsdKmp7e72V2ZlZItVpmJ2l+YAvgwKzd+Az4rJq2ylnFcSJwNbAksDRwVbkV6iJiYET0i4i5I2JQeu7gbGaF09KisjdJwySNLdmGlTS1PPAWcLGkxyRdKGneTrrtUjkz6P2B9SLiEwBJpwOPAWd29gZJq0XEM5KGdPR6RIyrZrBmZnmpZAYdESOBkZ28PICsbv6PImK0pHPI6hf9rNIxlVXNrt1xA9K+rhwNDANGlOwrTcFvU9bozMzqpIY56NeA1yJidHp+Hd0XmOtQV8WSfksWVN8FJkq6Mz3fDhjTTbsXSlo8IrZObR0AfBd4GTilmoGameWpVqs4IuJ1Sa9KWjUingW+DjxVTVtdzaDbVmpMBG4t2f/vMtq9gLTqQ9IWZOmQHwHrkn0s2K3ikZqZ5ajGqzh+BFyZVnC8CBxUTSNdFUu6qMqBAfSPiHfT4z2BkRFxPXC9pE7vZWhm1igtrZUsUOtaumfrBj1tp5ybxq5IdkXgGmRXBLYNYJUu3tZf0oCImEk2vS89w5nXncTNzKpWxIpB5QTLS4DTyArvf4tsqt7dt3I18E9JbwOfAg8ASFoJmFbtYM3M8tJawHKj5czp54mIOwEi4oWIOIksUHcqIk4HjiEL7puXVLPrR5abMTMrlAiVvdVLOTPo6alY0guSDgEmAwO7e1NEzHYyMSKeq3yIZmb5660pjp8A8wI/JstFzw98L89BAay2mhd62OwmffBGo4dgBXR+DdooYoqjnGJJbYutP+Tzov1mZn1KLVdx1EpXF6rcSBcnAyNi11xGZGbWAAXMcHQ5g/593UZhZtZgvSrFERH31HMgZmaN1Gvv6m1m1tcV8KbeDtBmZgBBL55BS5ozIqZ3f6SZWe8zs4ApjnLuqLKRpCeB59PzdST9LveRmZnVUaCyt3opZ+HfucBQ4B2AiHgc2DrPQZmZ1VtrBVu9lJPi6BcRk6Qv/NVoyWk8ZmYN0Vtz0K9K2ggISf3Jih25poaZ9Sm9dRXHoWRpjmWBN4C/p31mZn1GS2+cQUfEm8BedRiLmVnD1PaOV7VRzh1V/kQHl6lHxLAODjcz65Vae+MMmiyl0WYu4DvAq/kMx8ysMXpbsSQAIuKa0ueSLgcezG1EZmYN0FtPEra3PLBYrQdiZtZIreqFKQ5J7/H57L8f8C5wQp6DMjOrtyJe3NFlgFZ2dco6ZPchBGgtuQGsmVmfUcRVHF1e6p2C8W0R0ZI2B2cz65NaUdlbvZRTi2O8pPVyH4mZWQNFBVu9dHVPwgERMRNYDxgj6QXgY0Bkk+shdRqjmVnuipji6CoH/QgwBNipTmMxM2uY3rbMTgAR8UKdxmJm1jAtvWwGvaikozt7MSLOymE8ZmYN0dtm0P2B+aCAF6ibmdVYbwvQUyPi1LqNxMysgWp9S8JUP38sMDkihlbTRrc5aDOzZpDDDPpI4GlgULUNdLUO+uvVNmpm1tu0VLB1R9LSwLeBC3sypk4DdES825OGzcx6k1aVv0kaJmlsyda+Pv7ZwPH0cGJeTTU7M7M+p5JIGhEjgZEdvSZpKPBmRDwqaauejMkB2syMmuagNwN2krQD2U1OBkm6IiL2rbShcmpxmJn1ebWqxRERP42IpSNiMNn9XO+tJjiDZ9BmZkDvq8VhZtY08ijYHxH3AfdV+34HaDMzoLWAt411gDYzo/dd6m1m1jSKN392gDYzAzyDNjMrrJkq3hzaAdrMDKc4zMwKyykOM7OC8jI7M7OCKl54doA2MwOc4jAzK6yWAs6hHaDNzPAM2syssMIzaDOzYiriDNoF+wvsl+eczCNP/53bHxjV6KFYwXxzu62YOOF+nnnqQY4/7vBGD6dPaCXK3urFAbrArv/LzRy05xGNHoYVTL9+/Tj3nNMZuuO+fGWdrdlzz11YffWVGz2sXq9Wd1SpJQfoAhvz8Djef29ao4dhBbPRhuvxwgsv89JLrzBjxgxGjfobO+34zUYPq9ebSZS91YsDtFkvs+RSi/Pqa1NmPX9t8lSWXHLxBo6ob4gK/quXXE8SSloFOA5YrrSviNimk+OHAcMAFpl3GQbNtUiewzMzm6WIJwnzXsVxLXAB8CfKuOVXRIwERgKsuMiQ4q15MSuAKZNfZ5mll5z1fOmllmDKlNcbOKK+oRmX2c2MiD/k3IdZUxkzdjwrrbQ8gwcvw+TJr7PHHjuz3/5eydFTRZxB552DvlnSYZKWkLRQ25Zzn33G2SPP4Lo7LmH5lZbjwSduZ/d9dm70kKwAWlpaOPKok7jt1quY8MR9XHfdzTz11HONHlav1xJR9lYvihw7k/RSB7sjIlbo7r1OcVhHJn3wRqOHYAU087PJ6mkbey/3nbJjzlWTbuxxf+XINcUREcvn2b6ZWa0UMQeda4pD0jySTpI0Mj1fWdLQPPs0M6tGawVbveSdg74Y+AzYND2fDJyWc59mZhVrxku9V4yIXwMzACLiE6AuuRszs0o03YUqwGeS5iZdvi5pRWB6zn2amVWsnqszypV3gD4ZuANYRtKVwGbAgTn3aWZWsaa7aWxE3C1pHLAxWWrjyIh4O88+zcyqUcQLVfKuxTEkPZyavi4raX5gUkTMzLNvM7NK1Cq3LGkZ4DJgMbL07siIOKeatvJOcZwPDAGeIJtBrwVMBOaXdGhE3JVz/2ZmZalhimMmcExEjJM0EHhU0t0R8VSlDeW9imMKsF5EbBAR6wPrAS8C3wB+nXPfZmZli4iyt27amRoR49LjD4GngaWqGVPeAXqViJjY9iT9BVktIl7MuV8zs4q0EGVvkoZJGluyDeuoTUmDySamo6sZU94pjomS/gD8JT3fE3hK0pyktdFmZkVQSYqjtDRyZyTNB1wPHBURH1QzprwD9IHAYcBR6flDwLFkwXnrnPs2MytbLQvHSZqDLDhfGRE3VNtO3svsPgVGACMkDWnLyyQf5dm3mVklanWSUJKAi4CnI+KsnrRVz3sSXljHvszMKlLDS703A/YDtpE0Pm07VDOmvFMcpVyDw8wKq1aXekfEg9Qo3tUzQA+vY19mZhUp4qXeedeDPrjtcUT8VVJ/SSfn2aeZWTWasdzo1yXdlu5JuCbwb2Bgzn2amVWsVheq1FLeqzj2lrQn8CTwMbB3RDyUZ59mZtVoxhTHysCRZOsBJwH7SZonzz7NzKrRjAX7bwYOj4h70trAo4ExwJo592tmVpGWKF7B0bwD9EZtlzhGlrgZIenmnPs0M6tYPXPL5colQEvaJiLuBbbNJs6zeS6Pfs3MqlXEHHReM+gtgXuBHTt4LYCqr003M8tDPXPL5colQEfEyenrQXm0b2ZWa60FTHHksopD0iUljw/Iow8zs1oq4iqOvJbZrVPy+Mic+jAzq5mWaC17q5e8ctDF+6xgZtaFIqY48grQS0s6l6yiU9vjWSLixzn1a2ZWlaY5SQgcV/J4bE59mJnVTNPMoCPi0jzaNTPLSzPNoAGQtAFwIrBcaV8RsXae/ZqZVaolWho9hNnkfan3lWTpjieB4l3obmaWNM2l3iXeioibcu7DzKzHmulS7zYnS7oQuAeY3razJ7chNzPLQzPOoA8CVgPm4PMUh2txmFnhNM0qjhIbRsSqOfdhZtZjTbeKA/iXpDUi4qmc+zEz65FmLNi/MTBe0ktkOWiR1e73MjszK5RmzEFvn3P7ZmY10TQ5aEmD0q2uPsyjfTOzWmumGfRVwFDgUbJVG6X3vQpghZz6NTOrStOsg46Ioenr8nm0b2ZWa00zg5a0HPB+RExLz7cGdgFeBs6LiM/y6NfMrFpFXMWR1x1VRgHzAkhaF7gWeAVYFzg/pz7NzKrWGlH2Vi955aDnjogp6fG+wJ8jYoSkfsD4nPo0M6taEVMcec2gS08KbkNWi4OIAn6GMDOjtjeNlbS9pGcl/UfSCdWOKa8Z9L2SRgFTgQWBewEkLQE4/2xmhVOrGbSk/sB5wDeA14Axkm6q5orqvAL0UcCewBLA5hExI+1fnKyAv5lZodQwt7wR8J+IeBFA0l+AnYFiBOjI/hT9pXSfpKERcUu5bbzw9jh1f1RzkDQsIkY2ehxWLP69qK2Zn00uO+ZIGgYMK9k1suRnsRTwaslrrwFfrWZMeeWgO3JqHfvqa4Z1f4g1If9eNEhEjIyIDUq2XP5Q1jNAe0ZsZs1gMrBMyfOl076K5RKgJW3cwe4f5tGXmVnBjAFWlrS8pC8BewFV3fovrxn0bBejRMQjOfXVDJxntI7496KAImImcARwJ/A0MCoiJlbTlvJYnC1pXEQMqXnDZmZNJK8A/T5wf2evR8RONe/UzKyPyWsd9FvAiJzaNjNrCnnloD+MiH92tuXUZ91JOlHSRElPSBov6auSLpS0RgVtbCVpWnr/eEl/T/sPkbR/fqO3ckgKSSNKnh8r6ZSS58MkPZO2RyRtnvbfmH6e/2n38920Xfv9JJ0raYKkJyWNkbR8eu02SQtUMNYDJb1V0tdlaf+pkrbt4T+FNUBeM+iXc2q3MCRtQnZTgiERMV3SIsCXIuL7VTT3QFsN7TYRcUEtxmk9Nh3YVdKZEfF26QuShpKtTto8It6WNAT4q6SNIuI76ZitgGPb/3xL7AksCawdEa2SlgY+BoiIHaoY7zURcUTpjoj4eRXtWAHkMoOOiF0lfVnScEnXpW24pMXy6K9BlgDejojpABHxdkRMkXSfpA0AJH0k6Tdplv13SRul11+U1GUeXtIpabY2IM2qtkr7z5R0egfH95N0fprJ3Z1mX7ul19aX9E9Jj0q6M9VEIY3lV2nm95ykr9X2n6hPmEm2WuInHbz238BxbYE7IsYBlwKHV9D+EsDUtkJiEfFaRLwHIOllSYtIGpx+rpekn9OVkraV9JCk5yVt1FUH6X27SZo/FfBZNe2/WtIPOjh+HkmjJD2VPgmMLvmd3k7Sw5LGSbpW0nwlYx2e9j8pabUK/g2sE3mtg96MbC0gwGVpAxidXusL7gKWSf/DnC9pyw6OmRe4NyLWJLs/42lkBVS+wxevrPxaycfSL9QqSUt2DgT+kD6mbg8M76CvXYHBwBrAfsAmAJLmAH4H7BYR6wN/BkoD/ICI2IisfsrJFXz/zeQ8YB9J87fbvybZbd1KjU37yzUK2DH97EdIWq+T41YiO6+zWtr2BjYHjgX+p+S4PUt+lw4qbSDdQOMI4BJJewELRsSfOujrMOC9iFgD+BmwPkD6lHgSsG1apTUWOLrkfW+n/X9I47IeyivFMQLYJSIeK9l3k6QbgT9S5XXpRRIRH0laH/gasDVwjWYvK/gZcEd6/CQwPSJmSHqSLJi2mS3F0a6viZIuB24BNunkjjSbA9emmdjrkv6R9q8KrAXcLQmgP1mVwTY3pK+PthuTJRHxQcrn/hj4tMZtv5ZmtNuk7R5Ju0fEPe0OfSkingSQNBG4JyKig9+l2VIc7fq7W9LuZH901unksM2Bc9LxEyQ9kfZvTDYBeCj9Ln0JeLjkfaW/S7t28W1bmfIK0IPaBWcAImK8pIE59Vl3EdEC3Afcl/5HOaDdITPi83WMrWT5TFKusdJ/+68A7wNfBpD0VbI/dgBd5RgFTIyITTp5fXr62kJ+vw99wdnAOODikn1Pkc0u7y3Ztz7Q6UUJ7X9uEXFTSpPdDtwu6Q2y28O1D9DTSx63ljxvpYKfm7KbZqwOfEJWCvg1Sd/h809PXZ1DEXB3RPxXJ6/7d6nGcivYL2nBDnYulGOfdSVpVUkrl+xaF5iUU1+7AgsBWwC/k7RARIyOiHXTdhPwEPDdlIteDNgqvf1ZYFFlJzWRNIekSj6CGxAR75KlIw4u2f1r4FeSFoZZt3c7kC5u69b+5yZpiKQl0/v7AWuT0+9R8hOyq9v2Bi6WNEdE3FgyprFkv0t7pDGtQTY5APg3sJmkldJr80paJcexNr28guVvgbskbSlpYNq2Ipsl/DanPuttPuDSdCLlCbKPfqfUupOU9/sl8P2IeA74PenjZzvXk5U1fAq4gmy2Ny2lQ3YjCySPk91ybNMO3m/dGwEs0vYk/WH8M/AvSc8AfwL2jYipnby/I18GbpY0AXiC7KTk72s35M+lVMr3gWMi4gGyi8lO6uDQ88n+qD9Fdt5kItnv0ltkf4CuTr/zD5Plwy0nuVxJCLOWIB3P5ydMJgK/iYibc+nQkDRfyo0vDDwCbBYRrzd6XNa7KLsjyBwR8X+SVgT+DqzaybkPy1FuAdrqT9J9wAJkJ29+HRGXNHRA1iul80T/AOYgyzv/d0Tc3thRNae8anF0ddIqIuIXNe/UzKyPyStAH9PB7nnJTrAsHBHz1bxRggAPAAADmElEQVRTM7M+JvcUR/q4dCRZcB4FjIiIN3Pt1MysD8htrWJaUnc0sA/Z5a9D2i5hNTOz7uV1qfdvyC71/hD4SkSc4uDce0hqSZcKT0j1FubpQVtbSbolPd6pg6stS49dQNJhVfRxiqTZLi3ubH+7Yy5RqllSZl+D05I4s9zltQ76GLIKXScBUyR9kLYPJX2QU59WO5+mixbWIrtc/ZDSF5Wp+HcnXTX3yy4OWYCsDoSZkV81u34RMXdEDIyIQSXbwIgYlEeflpsHgJXSzPHZVJNiAlmhqM4qm22vrPraOEpqMiirV/z79HgxZZXSHk/bpmQX5KyYZu+/Sccdp6ya3xOShpe0daKyQlUPktUb6ZKkH6R2Hpd0fbtPBdtKGpvaG5qO76+sEmFb37Pd9FjSmsoqAY5Px6zc/hiznugTl11bPlK9kG+RFXoCWBk4P1Xn+5gOKptJmovsirodyepSLN5J8+cC/4yIdYAhZBcynQC8kGbvx0naLvW5Edml9OtL2kJZkaq90r4dgA3L+HZuiIgNU39P88VLtgenPr4NXJC+h4PJrp7bMLX/A6VC+iUOAc6JiHWBDciu5DSrGRc0sY7MLWl8evwAcBFZympSRPw77e+sstlqZJXXngeQdAUwrIM+tgH2h1lFp6Zp9vot26WtrfDWfGQBeyBwY0R8kvoo55b2a0k6jSyNMh/ZHZfbjEpVAJ+X9GL6HrYD1i7JT8+f+n6u5H0PAycqK7J/Q9v3bFYrDtDWkU/TrHCWFIQ/Lt1FB5XNUsGgWhFwZkT88Qs7paOqaOsSshK4j0s6kM+LSQG0X2saqe8fRURpIEfS4FkHRVwlaTTZzPs2ST+MiNLKdmY94hSHVauzymbPAINTDQeAzkpT3gMcmt7bX1kx/A/JZsdt7gS+V5LbXkrSl8mK/Owiae60zn7HMsY7EJiq7AYG+7R7bXdlVQBXBFYgqwB4J3BoOh5Jq0iat/RNklYAXoyIc4G/kVWiM6sZz6CtKhHxVpqJXi1pzrT7pIh4TtIw4FZJn5ClSDqqAX4kMFLSwWT1gw+NiIeV3cZpAnB7ykOvDjycZvAfkVWLGyfpGuBx4E0+v3tPV34GjCa74/zodmN6hay41CDgkFQk6EKy3PQ4ZZ2/RVanudQewH6SZgCvA2eUMQ6zsrlYkplZQTnFYWZWUA7QZmYF5QBtZlZQDtBmZgXlAG1mVlAO0GZmBeUAbWZWUP8PpcBVWRvwZ78AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"SimFix\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=1)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "\n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['SimFix-gen','NOT-SimFix-gen']); ax.yaxis.set_ticklabels(['SimFix-gen' , 'NOT-SimFix-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "    auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "    print('ROC AUC: %f' % auc)\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: -0.071429\n",
      "ROC AUC: 0.785714\n",
      "Accuracy: 0.866667\n",
      "Precision: 0.000000\n",
      "Recall: 0.000000\n",
      "F1 score: 0.000000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEdCAYAAAARhuyGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcHFW5//HPN6wSEiBhXwRBwJVNBFlE1ggYAREEBFnEG0GuF0VQEUTEhU24yiYEkR0u8IMIIlsIIIsIITHsyo6QBGIIRAIYkszz+6NOQ09neqa6p2uma/J986rXdJ+uqnMm0zx9+tQ5TykiMDOz8hrU3w0wM7PecSA3Mys5B3Izs5JzIDczKzkHcjOzknMgNzMrOQdy6zVJH5D0R0kzJV3Ti/PsK+m2VratP0i6WdIB/d0OW3A4kC9AJH1V0kOSZkmamgLOli049R7ACsDwiNiz2ZNExOURMaIF7elE0taSQtKYmvL1U/ldOc9zvKTLetovInaKiIubbK5ZwxzIFxCSjgB+DfySLOh+EDgH2LUFp18deCoi5rbgXEX5F7CZpOFVZQcAT7WqAmX8/5T1Ob/pFgCSlgJOAA6LiOsi4q2ImBMRf4yIo9I+i0n6taQpafu1pMXSa1tLelnS9yRNS735g9JrPwWOA/ZKPf2Da3uuktZIPd+F0/MDJT0n6U1Jz0vat6r83qrjNpc0Pg3ZjJe0edVrd0n6maT70nluk7RsN/8M7wJ/APZOxy8E7AVcXvNv9RtJL0n6t6QJkj6byncEflT1ez5c1Y5fSLoPeBtYM5V9I73+W0nXVp3/ZEnjJCn3H9CsBw7kC4bNgMWBMd3scwzwGWADYH1gE+DYqtdXBJYCVgEOBs6WtExE/ISsl39VRCwZERd01xBJg4EzgJ0iYgiwOTCpi/2GAX9K+w4HTgf+VNOj/ipwELA8sChwZHd1A5cA+6fHnwceA6bU7DOe7N9gGHAFcI2kxSPilprfc/2qY74GjAKGAC/WnO97wCfTh9Rnyf7tDgjnxrAWciBfMAwHpvcw9LEvcEJETIuIfwE/JQtQFXPS63Mi4iZgFrBuk+3pAD4h6QMRMTUiHu9iny8AT0fEpRExNyKuBP4OfLFqnwsj4qmIeAe4miwA1xURfwGGSVqXLKBf0sU+l0XEa6nO04DF6Pn3vCgiHk/HzKk539tk/46nA5cB346Il3s4n1lDHMgXDK8By1aGNupYmc69yRdT2XvnqPkgeBtYstGGRMRbZEMahwBTJf1J0kdytKfSplWqnr/SRHsuBf4b2IYuvqFIOlLSk2k45w2ybyHdDdkAvNTdixHxAPAcILIPHLOWciBfMNwPzAZ262afKWQXLSs+yPzDDnm9BSxR9XzF6hcj4taI2AFYiayXfX6O9lTaNLnJNlVcCnwLuCn1lt+Thj6+D3wFWCYilgZmkgVggHrDId0Ok0g6jKxnPyWd36ylHMgXABExk+yC5NmSdpO0hKRFJO0k6ZS025XAsZKWSxcNjyMbCmjGJGArSR9MF1qPrrwgaQVJu6ax8tlkQzQdXZzjJmCdNGVyYUl7AR8DbmyyTQBExPPA58iuCdQaAswlm+GysKTjgKFVr78KrNHIzBRJ6wA/B/YjG2L5vqRuh4DMGuVAvoBI471HkF3A/BfZcMB/k83kgCzYPAQ8AjwKTExlzdQ1FrgqnWsCnYPvoNSOKcAMsqB6aBfneA0YSXax8DWynuzIiJjeTJtqzn1vRHT1beNW4BayKYkvAv+h87BJZbHTa5Im9lRPGsq6DDg5Ih6OiKfJZr5cWpkRZNYK8sVzM7Nyc4/czKzkHMjNzErOgdzMrOQcyM3MSs6B3Mys5Lpb6dev5kx/ztNpbD5rrdOKZI020PxzxqO9TkLWSMxZZNk12yrpWdsGcjOzPtUxr79b0DQHcjMzgOhqgXE5OJCbmQF0OJCbmZVauEduZlZyLeyRS/o9Wa6gaRHxiVR2Klk+/XeBZ4GDIuKNLo59AXgTmAfMjYiNe6rP0w/NzADmzcm/9ewiYMeasrHAJyJiPbLEbEfXHlRlm4jYIE8QBwdyM7NMdOTfejpVxN1k2T2ry26rujnLX4FVW9V0B3IzM8iGVnJukkZJeqhqG9VgbV8Hbq7zWgC3pZt/5zqvx8jNzGjsYmdEjAZGN1OPpGPIbmByeZ1dtoyIyZKWB8ZK+nvq4dflHrmZGTTUI2+WpAPJLoLuG3VuBhERk9PPaWT3ld2kp/M6kJuZQUvHyLsiaUeyO13tUnu/2Kp9BksaUnkMjAAe6+ncHloxM4O8s1FykXQlsDWwrKSXgZ+QzVJZjGy4BOCvEXGIpJWB30XEzsAKwJj0+sLAFRFxS0/1OZCbmUFL55FHxD5dFF9QZ98pwM7p8XPA+o3W50BuZgbOtWJmVnrOtWJmVm4RTmNrZlZu8+b2vE+bciA3MwOPkZuZlZ7vEGRmVnLukZuZlZxnrZiZlZx75GZmJTfXs1bMzErN88jNzMrOY+RmZiXnMXIzs5Jzj9zMrOS8RN/MrOQ8tGJmVnIeWjEzKzkHcjOzkvPQiplZyblHbmZWcp61YmZWch5aMTMrOQ+tmJmVnAO5mVnJRfR3C5rmQG5mBu6Rm5mVnmetmJmVnHvkZmYl5zFyM7OSc4/czKzkHMjNzMot5pX35suD+rsBZmZtoaMj/9YDSb+XNE3SY1VlwySNlfR0+rlMnWMPSPs8LemAPE13IDczgyzXSt6tZxcBO9aU/RAYFxFrA+PS804kDQN+AmwKbAL8pF7Ar+ZAbmYG0BH5tx5ExN3AjJriXYGL0+OLgd26OPTzwNiImBERrwNjmf8DYT4eIzczg7642LlCRExNj18BVuhin1WAl6qev5zKuuVA3maO/eXp3H3fgwxbZmn+cNm5AJw5+hLuuPd+BmkQw5ZZil8c8z2WX254P7fU+supZ57AdiO24rXpM9hhi937uzkDRwOBXNIoYFRV0eiIGJ33+IgISS2buO6hlTaz2847cO7pP+9UdtC+X2bMJb/l2ovP5nNbbMpvL7yin1pn7eCaK65n/z0P7e9mDDzz5uXeImJ0RGxcteUJ4q9KWgkg/ZzWxT6TgdWqnq+ayrrlQN5mNt7gkyw1dEinsiUHD37v8Tvv/Aepr1tl7eTB+yfwxusz+7sZA08Lx8jruAGozEI5ALi+i31uBUZIWiZd5ByRyrpV6NCKpN2Bk4HlAaUtImJokfUORL857yJuuGUcQwYP5vdnntTfzTEbeFp4hyBJVwJbA8tKeplsJspJwNWSDgZeBL6S9t0YOCQivhERMyT9DBifTnVCRNReNJ1P0T3yU4BdImKpiBgaEUO6C+KSRkl6SNJDv7vkyoKbVi6Hf/NAxo25lC+M2IYrrv1jfzfHbOBp7ayVfSJipYhYJCJWjYgLIuK1iNguItaOiO0rAToiHoqIb1Qd+/uI+HDaLszT9KID+asR8WTenavHnb6x/z5Ftqu0Ro7Yhtvvuq+/m2E24ERHR+6t3RQ9a+UhSVcBfwBmVwoj4rqC6x1QXnxpMquvls1AuuOe+/nQ6qv2c4vMBqDmx777XdGBfCjwNtmAfUUADuR1HPWTkxj/t0d4441/s91u+/Gtg7/GPfeP54V/vowGiZVXXJ7jjvp2fzfT+tGZ55/MZlt8mmWGL80Dj93O6SedzVWXjenvZpVfiXOtKNo0B++c6c+1Z8OsX621zq793QRrQ/+c8Wiv53K9dfw+uWPO4OOvbKu5Y4WOkUtaR9K4SuIYSetJOrbIOs3MmlL89MPCFH2x83zgaGAOQEQ8AuxdcJ1mZo1rbdKsPlX0GPkSEfGgOq9gKe8dTs1s4GrDnnZeRQfy6ZLWIrvAiaQ9gKndH2Jm1vdibnkvdhYdyA8DRgMfkTQZeB7Yr+A6zcwa5x551yLiOWB7SYOBQRHxZpH1mZk1rQ3HvvMqOtfKETXPAWYCEyJiUpF1m5k1xD3yujZOWyU5yEjgEeAQSddExCkF129mlks4kNe1KrBRRMwCkPQT4E/AVsAEsqRaZmb9z4G8ruWpyrFCNp98hYh4R9LsOseYmfU9z1qp63LgAUnXk+UiHwlckS5+PlFw3WZm+blH3rWI+Jmkm4EtUtEhEfFQerxvkXWbmTWiXfNO5VH4rd5S0vTfAO9UBXEzs/biXCu5HNKHdZmZNabEgbzoMfJqbZX20cysmqcf5vPFPqzLzKwxc8sbyIvOR36ppKUAIuJlSatLGldknWZmzYiOyL21m6J75PeSTT88AlgFOAr4XsF1mpk1rg0DdF5FTz88T9LjwJ3AdGDDiHilyDrNzJpS3pxZhQ+tfA34PbA/cBFwk6T1i6zTzKwZHlqp78vAlhExDbhS0hjgYmCDgus1M2tIlPhiZ9FDK7vVPH9Q0iZF1mlm1pQSD60UEsglfT8iTpF0Rp1d/qeIes3MmlXi+0oU1iN/Mv2cUND5zcxaayAHckm7A2Mj4k1JPwQ2An7Zwx1+bgaIiItb00wzs2KVuUeeZ9bK8SmIbw7sTJaa9twejnmw8kDSmb1on5lZ3+hoYGszeQJ5Jdv6SOC8iLgeWKyHY6rzqmxRdy8zszbRMTf/1m7yjJFPlXQ2sCOwsaRF6fkDoLzzeMxsgVTmoZU8gfwrZEMqZ0bE65JWBn7YwzEfkfQIWc98rfSY9DwiYr2mW2xmVoQob4LWuoFc0tCqp7dUlc0C7uvhvB/tfdPMzPpOq3rkktYFrqoqWhM4LiJ+XbXP1sD1wPOp6LqIOKHZOrvrkT9ONkRS/TFVeR7AB+sdGBEvNtsgM7P+EB2t6ZFHxD9Iq9clLQRMBsZ0ses9ETGyFXXWDeQRsVpvT56mLp4MLE/2AVAZWhna7YFmZn2soDHy7YBni+7c5kqaJWlvST9Kj1eV9Kmc5z8F2CUiloqIoRExxEHczNpRxzzl3hqwN3Blndc2k/SwpJslfbw3be8xkEs6C9gG+Foqepue55FXvBoRT/a8m5lZ/4oO5d4kjZL0UNU2qvZ8aYbfLsA1XVQ3EVg9ItYHzgT+0Ju255m1snlEbCTpbwARMSM1sK40pALwkKSrUiNnV16PiOuabbCZWRGigUnTETEaGN3DbjsBEyPi1S6O/3fV45sknSNp2YiYnr8V78sTyOdIGkSaGy5pOD2vbaq+P+fbwIiq5wE4kJtZW2nVxc4q+1BnWEXSimQjFpEywg4CXmu2ojyB/GzgWmA5ST8lm1f+0+4OiIiDmm2QmVl/aGUglzQY2AH4ZlXZIQARcS6wB3CopLnAO8DeEY18J+isx0AeEZdImgBsn4r2jIjH8pxc0sXA4RHxRnq+DHBaRHy92QabmRWhwYuY3YqIt4DhNWXnVj0+CzirVfXlTWO7EDCHbFikkdvDrVcJ4gBpZeiGDRxvZtYnosQrO/PMWjmGbJxnZWBV4ApJR+c9f+qFV841jOJvL2dm1rDoyL+1mzxBdX9gw4h4G0DSL4C/ASfmOPY04H5Jlek3ewK/bKahZmZF6ihxjzxX9sOa/RZOZT1K4+sPAdumot0j4onGmmhmVrwyD610lzTrf8nGxGcAj0u6NT0fAYzPc3JJl0bE14AnuigzM2sbBUw/7DPd9cgrM1MeB/5UVf7XBs7fadlpSiCTd3m/mVmfaeWslb7WXdKsC5o9aboY+iPgA5L+zfsZFN+l59VQZmZ9bkCPkUtaC/gF8DFg8Up5RKxT75iIOBE4UdKJEZF3houZWb8p8xh5njnhFwEXkvWqdwKupnPS9O4cI2k/ST8GkLRaWo5qZtZWIvJv7SZPIF8iIm4FiIhnI+JYsoCex9nAZsBX0/NZqczMrK10hHJv7SbP9MPZKWnWsylXwGRgSM7zb1qTOfH1njInmpn1h44BOmul4rvAYOB/yMbKlwLy5kqZk2aqVDInLkfPmRMBWGudXXNWYQuSKbNm9HcTbIBqx552XnmSZj2QHr7J+zeXyOsMsnvVLZ9WhO4B/LjBc5iZFa7MFzu7WxA0htST7kpE7F7vtap9Lk+ZE7cju1i6m+8YZGbtaKD2yHudYlHSwWk++t+ryk6KiB/29txmZq3UhpNRcutuQdC4Fpz/y5L+ExGXA0g6m6q56GZm7WKg9shb4cvADZI6gB2BNyLi4ILrNDNr2DwH8s5S3vGKbwDXA/cCP5U0LCI89cDM2kqwAARySYtFxOycu08gG3JS1c+d0wawZiONNDMrWkeJB8nz3CFoE0mPAk+n5+tLOrOHw/YCtoiID0XEmsDxZNkUbwQ27l2TzcxarwPl3tpNniX6ZwAjgdcAIuJhYJsejjkXmA0gaSuyuwldDMzE2Q/NrA0Fyr21mzxDK4Mi4kWpU+Pn9XDMQlXj4HsBoyPiWuBaSZOaaKeZWaHa8FacueUJ5C+ljIWRltt/G3iqh2MWkrRwRMwlWww0qsE6zcz61Lw27GnnlSeoHko2vPJB4FXg9lTWnSuBP0uaDrwD3AMg6cNkwytmZm1lQPfII2IasHcjJ42IX0gaB6wE3BbxXgbfQWQ9ejOzttKOY9955blD0Pl0sXo1IkZ1sXv16/Pd2zMiehqSMTPrFyXOYptraOX2qseLA18CXiqmOWZm/aMdpxXmlWdopdNt3SRdSrZK08xswOhpKl47a2YGyYeAFVrdEDOz/tShAdwjl/Q674+RDwJmAE5Da2YDSolX6HcfyJWtAlqf7D6dAB1VM1DMzAaMMk8/7HaJfgraN0XEvLQ5iJvZgNSh/Fu7yZNrZZKkDQtviZlZPypz0qzu7tlZWWK/ITBe0rPAW6TUtBGxUR+10cyscPNaGJ8lvUB2w/p5wNyI2LjmdQG/IUvt/TZwYERMbLa+7sbIHwQ2AnZp9uRmZmVRwBj5NhExvc5rOwFrp21T4LfpZ1O6C+QCiIhnmz25mVlZ9PEFwF2BS9J1x79KWlrSShExtZmTdRfIl5N0RL0XI+L0Zio0M2tHLb6IGcBtkgI4LyJq78OwCp1XyL+cyloeyBcCloQ2HNk3M2uxRoZWJI2ic3ru0TXBesuImCxpeWCspL9HxN0taWgXugvkUyPihKIqNjNrJ40E8hS0697tLCImp5/TJI0BNgGqA/lkYLWq56vy/nqdhnU3/dA9cTNbYMxT/q07kgZLGlJ5DIwgu2dxtRuA/ZX5DDCz2fFx6L5Hvl2zJzUzK5sWzlpZARiTbo+5MHBFRNwi6RCAiDgXuIls6uEzZNMPD+pNhXUDedU9N83MBrxWzVqJiOfIUpvUlp9b9TiAw1pUpe+faWYG7bn0Pi8HcjMzyp00y4HczIwF78YSZmYDjodWzMxKzkMrZmYlV+abLTiQm5kBHSUO5Q7kZmZ4aMXMrPQ8a8XMrOQ8a8XMrOQ8Rm5mVnLlDeMO5GZmgC92mpmVnodWzMxKzrNWzMxKzj1yM7OSK28YdyA3MwN8sdPMrPSixH3yQf3dAKvv1DNPYOI/7mLsfdf1d1OszXx+xNY8/tjd/P2Je/n+US279eMCbS6Re2s3DuRt7Jorrmf/PQ/t72ZYmxk0aBBn/OYXjPzifnxy/W3Ya6/d+OhH1+7vZpVeNLC1GwfyNvbg/RN44/WZ/d0MazObfHpDnn32BZ5//p/MmTOHq6++nl2++Pn+blbpdRC5t3bjQG5WMiuvsiIvvTzlvecvT57Kyiuv2I8tGhg6GtjaTaEXOyVtARwPrJ7qEhARsWaR9ZqZNarMFzuLnrVyAfBdYAI5Fk5JGgWMAlhmiZVZcrFhxbbOrISmTH6F1VZd+b3nq66yElOmvNKPLRoY2rGnnVfRQyszI+LmiJgWEa9Vtno7R8ToiNg4IjZ2EDfr2viHJvHhD3+INdZYjUUWWYSvfGVX/njjbf3drNKbR+Te2k3RPfI7JZ0KXAfMrhRGxMSC6x0Qzjz/ZDbb4tMsM3xpHnjsdk4/6WyuumxMfzfL+tm8efM4/DvHctOfrmChQYO46OKreOKJp/q7WaXXEe0XoPNSFNh4SXd2URwRsW1Px35w2CfL+69qhZkya0Z/N8Ha0Nx3J/f6/j77rb577phz2YvXtdX9hArtkUfENkWe38ysVdpxWmFehY6RS1pB0gWSbk7PPybp4CLrNDNrRjTwX7sp+mLnRcCtQOUS+1PAdwqu08ysYV6iX9+yEXE1aWZPRMyl3PnbzWyAKnOPvOhZK29JGk5KTyDpM4DXnJtZ2/E88vqOAG4A1pJ0H3AJ8O2C6zQza1hE5N66I2k1SXdKekLS45IO72KfrSXNlDQpbcf1pu1Fz1qZKOlzwLpky/P/ERFziqzTzKwZLZy1Mhf4Xop/Q4AJksZGxBM1+90TESNbUWHRuVZ2rylaR9JM4NGImFZk3WZmjWjV0EpETAWmpsdvSnoSWAWoDeQtU/QY+cHAZkBlYdDWZHlXPiTphIi4tOD6zcxymVfAKLmkNYANgQe6eHkzSQ8DU4AjI+LxZuspOpAvDHw0Il6FbF452Tj5psDdgAO5mbWFRla5Vyf4S0ZHxOiafZYErgW+ExH/rjnFRGD1iJglaWfgD0DTdwcpOpCvVgniybRUNkOSx8rNrG000h9PQXt0vdclLUIWxC+PiPnu1Vgd2CPiJknnSFo2IqY30uaKogP5XZJuBK5Jz7+cygYDbxRct5lZbq2aHy5JZCm8n4yI0+vssyLwakSEpE3IZhDWzQzbk6ID+WHA7sCW6fklwLWRfYdxHhYzaxstnLWyBfA14FFJk1LZj4APAkTEucAewKGS5gLvAHtHLzIYFj39MMi+XlwraWRE3FhkfWZmzWpVJtiIuJdsunV3+5wFnNWSCunbe3ae0Id1mZk1ZB4dubd2U/TQSrW2yt9rZlatzDeW6MtA/s0+rMvMrCHlDePF5yP/maSFASLiQUlDJV1YZJ1mZs3oIHJv7aboMfKFgQckrSdpB2A82cpOM7O2UuZAXvSslaMl3U62PPV1YKuIeKbIOs3MmjEv2u8iZl5FD61sBZxBNmPlLuBMSSt3e5CZWT/wjSXq+xWwZyV9Y8qGeAfwkYLrNTNrSKvmkfeHogP5ZhHx3q3dIuI6SX8uuE4zs4a149h3XoUEckn7RcRlwOFZ2oH5dJl/wMysv7hHPr/B6eeQgs5vZtZS7pHXiIjz0s+fFnF+M7NW86yVGpJuq3p8dBF1mJm1UplnrRQ1/XC5qsd7FlSHmVnLdETk3tpNUWPk7febmpl1ox172nkVFcjXlHQDWcbDyuP3RMQuBdVrZtaUduxp51VUIN+16vGvCqrDzKxl3COvERFe9GNmpVLmWSuFruyUtAVwPLB6qktkd4Bbs8h6zcwaFQ7kdV0AfJcsde28HvY1M+s3XhBU38yIuLngOszMes1L9GtI2ig9vFPSqcB1wOzK6xExsYh6zcya5R75/E6reb5x1eMAti2oXjOzpszr8Bh5JxGxTRHnNTMrSpmnHxZ9h6BfSlq66vkykn5eZJ1mZs2IiNxbuyn65ss7RcQblScR8Tqwc8F1mpk1zDdfrm8hSYtFxGwASR8AFiu4TjOzhrVjTzuvogP55cA4SRem5wcBlxRcp5lZw5xrpY6IOFnSw8D2qehnEXFrkXWamTXDS/TrkHRyRPwAuKWLMjOztlHmoZWiL3bu0EXZTgXXaWbWMN9YooakQ4FvkeUif6TqpSHAfUXUaWbWG2WeR17U0MoVwM3AicAPq8rfjIgZBdVpZta0duxp51XI0EpEzIyIFyJiH2A1YNuIeBEYJOlDRdRpZtYbrVwQJGlHSf+Q9IykH3bx+mKSrkqvPyBpjd60veiVnT8BfgAcnYoWBS4rsk4zs2Z0REfurTuSFgLOJrse+DFgH0kfq9ntYOD1iPgw8L/Ayb1pe9EXO78E7AK8BRARU8jGyc3M2koLe+SbAM9ExHMR8S7wf3S+/SXp+cXp8f8DtpOkZtte9IKgdyMiJAWApMF5D/znjEeb/qUGGkmjImJ0f7fD2ovfF601593JuWOOpFHAqKqi0VV/i1WAl6peexnYtOYU7+0TEXMlzQSGA9MbbTcU3yO/WtJ5wNKS/gu4HfhdwXUORKN63sUWQH5f9JOIGB0RG1dt/fqBWvTKzl9J2gH4N7AucFxEjC2yTjOzfjaZbJJHxaqprKt9Xpa0MLAU8FqzFRa9snOndKu3sVVlh0TEuUXWa2bWj8YDa6cZepOBvYGv1uxzA3AAcD+wB3BH9GJpadFDKz+W9N7dgCR9n/kH/a1nHge1rvh90YYiYi7w38CtwJPA1RHxuKQTJO2SdrsAGC7pGeAIOq+3aZiKzC8gaVngRuAoYEfgI8A+6UqumZm1QKGBHEDS8mQXOScAX+/N1wczM5tfIYFc0pvQKXHBosDcVBYRMbTllZqZLaCKWqI/JCKGVm2LR8SSlfIi6iyCpGMkPS7pEUmTJG0q6YU0ZJT3HD/Ksc9FkvboonwNSe+kuivbopJ26WrZrzVOUkg6rer5kZKOr3o+StLf0/agpC1T+Zj093hG0syqv8/mNecfJOkMSY9JelTS+EqaCkmzGmjn0pK+lWO/uyRt3EX51jXtvD2VHyJp/7ztsPZUVPbD1YE3ImJmer4NsBvwAnB2GcbIJW0GjAQ2iojZKXgv2sSpfgT8shdNeTYiNqgpuyFt1nuzgd0lnRgRnRZjSBoJfBPYMiKmS9oI+IOkTSLiS2mfrYEjI2JknfPvBawMrBcRHZJWJa10btDSZBlFz2ni2Ip7atvpGWQDQ1GzVq4GBgNI2gC4BvgnsAG9eyP2pZWA6ZX7jUbE9JRiAMjuPyrp5rTQCUn7pR7bJEnnSVpI0knAB1LZ5Wm//VMP/2FJl1bVt5Wkv0h6rqveeTVJB0o6Kz2+vtKjkvTNSj1dHPPjlMTnXklXSjoyla8l6RZJEyTdI+kjqfyi1JPM1aYSm0s2++O7Xbz2A+CoSoCPiIlky6oPa+D8KwFTI7IEHRHxcroJ+XskLSvpfklfSM+PSj33RyT9NO12ErBWei+dmvb7QerlP5zeaxV7pvfiU5I+213jJB2fvoUsnOrcOpWfKOkXXew/SNI56RvKWEk3Vd4bkj4l6c/pvXSrpJVS+V2STs7bJmtCI/kFGshD8EjV418Bp6THg6pfa+cNWBKYBDxF9uHzuVT+ArAG2QXc/VMdNmwrAAAJM0lEQVTZR4E/Aouk5+dUvTar6pwfT+dbNj0fln5eRPZhN4gsyc4zqXwN4J3Ujklk32YADgTOSo9XAJ4BPpvOPayL3+XT6fjFyXLdPE3WiwQYB6ydHm9KNp+1bpsG2gbMAoamv+tSwJHA8em1GcBSNfvvClxX9Xxr4MZuzr9qOvck4DRgw5q6VwAeAHZIZSPIPliU/u1vBLZK74XHqo7dCfgLsETNe+ku4LT0eGfg9qp2zqx6Lx2Tyo+vei98nGy63PbA34BFu/h99gBuSm1bEXg9lS2S2rNc2m8v4Pfdtclb67aiFgRV5yzYlpT9MLKvlgVV2VoRMUvSp8gC5DbAVVXj0teTfThVer/bAZ8Cxqff7wPAtC5Ouy1wTbzfw6vOzf6HyHptT0haoaq8q6GV6na+Kuk44E7gS9F1vvctgOsj4j/AfyT9EUDSksDmwDVVf5fFcrRpQImIf0u6BPgfsg/OVp77ZUnrkv3ttyW7GfmeETGOLPiNAw6LiD+nQ0ak7W/p+ZLA2mTfaKttD1wYEW+neqr/7telnxPIPgAq5htaqWnr4+lb4o3AZtH1EOiWZO/hDuAVSXem8nWBTwBj03tpIWBqjjZZCxQVyO+QdDXZH3IZ4A6A9FWr7cfHKyJiHllv4i5Jj5KtxILsLkc7Sroism6GgIsj4uiuz5TL7KrHjX7afZJsee/KAJJWI/uGANDdGOggsmsZ9T4oetOmsvk1MBG4sKrsCbIP6Duqyj4FPF7vJJI2Bc5LT4+LiBsiG567GbhZ0qtk14vGkQ3rTAA+D1QCuYATI+K8mvOu0cDvUvm7zaPx/8c/CbwBLN/V79PNcQIej4jNCmiT9aCoMfLvkH0Cv0B2oWhOKl8ROKagOltK0rqS1q4q2gB4MT0+juwr5dnp+ThgD2Vz5pE0LF3wBZgjaZH0+A6y8cvhlf1a0M5NyL5mbwgcKelDEfFSRGyQtnPJPni+KGnx1AsfCVlPFHhe0p7pXJK0fm/bVEapR3s1WZ7oilOAk6v+XhuQDWvVvc4TEQ9U/dvfIGkjSZUP2EHAerz/Pgrg68BHJFVuSH4r8PX0d0LSKul99SadU0CPBQ6StETarxXvpd2BYWRDOWdKWrr29yF7L305jZWvQDZkA/APYDllkwSQtIikj/e2TZZPIZ+MqZf6f9VlkkZGxI1F1FeQJUlvZrKe0zNk2eYqX00PB34v6ZSI+L6kY4Hb0v+sc8guiL1INt75iKSJEbFvuoD0Z0nzyL4+H9hsAyUtBpwPHBQRUyR9L7Vp2/Q3ACAixku6AXgEeBV4lGy8FGBf4Lep/YuQ/d0ebrZNJXca2dJqAFIgXgX4i7JUzG8C+0XE1Hon6MLywPnpbwXwIHBWVR3zJO0D3CDpzYg4R9JHgfvTEMWsVOezku6T9Bhwc0QclT5YHpL0Ltm4dY9TXetRNivrJGC7iHhJ2cX03/D+t9CKa8mGEp8gS8M6EZgZEe+mi55nSFqKLLb8mm6+vVjrFL6y872KskC2UZ9UZvORtGQa918CuBsYFdksDLOGVL2XhpN9MG0REa/0d7sWZH05VjXQx1jb3Whlt5tanGw830HcmnVj+qa6KPAzB/H+V9QS/c9ExF9ryjaJiAdbXpmZ2QKuqEDuYRQzsz5SdD5yMzMrWFE98jfILqh1KSJ2qfeamZk1pqiLnf8im8plJZSmRj5K9v54EjigsoKwiXNtTUoqpezuKB+LiJPq7Ls08NWIaCgfj7JshbMi4ld5ymv2uYhsif3/y1nXGmn/TzTSRrMiFRXI36xacmzl805ltaeyJFyHAKdXXlQ2wVlpmXZuaUFJd1kbW5Hhz2yBU9QY+QsFndf63j3Ah5XlRv9HyknyGLCapBHKsvZNlHRN1WrEHZVlx5sI7F45kTpnbVxBWU7vh9O2OV1n+OsqE2AlV/xTku4ly/PRLUn/lc7zsKRrKysik+0lPZTONzLtv5CkU6vq/mYX5/y43s94+Yg6rwQ26zNFrezcPS0rPowsoxpkK7zOiYhXi6jTWk/SwmTL/29JRWuTDbP8Na0EPBbYPiLeSkvMj5B0Ctlq023JVsNeVef0ZwB/jogvSVqIbCXtD4FPVH0bGJHq3IRsHcINkrYiy+e9N1nahIXJVhdO6OHXuS4izk/n/TnZUvwz02trpDrWAu6U9GFgf7IVi59OqzLvk3Qbne98dQjwm4i4XNKiZImizPpcUTeW2AK4giwV6iWp+FPAA5L2jYj7iqjXWuYDkialx/eQ3fF7ZeDFqvUBnyFLb3tfWkq+KHA/2Q22n4+IpwEkXUaW2qDWtmTBspKcbKakZWr2qZcJcAgwpjJun9IP9OQTKYAvnc5za9VrV6dhoqclPZd+hxHAeno/D/tSqe6nqo67HzhG2c0irqv8zmZ9ragx8tOA3SLib1VlN0gaQ5ZJbdOC6rXWeG+MvCIF6+o72wgYGxH71OxXN+VuE+plAvxOE+e6iOw9+bCkA3k/2RN07mVXngv4dkRUB/xOWQgj4gpJDwBfAG6S9M2IqM6UaNYnihojH1oTxAGIiEl0zuBm5fVXYIs0DIGkwZLWAf4OrCFprbTfPnWOHwccmo5dKCVaqs3wVy8T4N3Absru0jQE+GKO9g4BpirLRLlvzWt7KsvmtxawJlkmv1uBQ9P+SFpH0uDqgyStCTwXEWeQ5ahfL0c7zFqusBtLSFom5r+l1TC8CGlAiIh/pZ7tlXo/s9+xEfGUpFHAnyS9TTY009WH9+Fk+V8OJstRfWhE3K/5M/x1lQlwoqSryLI0TgPG52jyj8nuxPOv9LO6Tf8kS/40FDgkIv4j6XdkY+cT0yydf5HlEa/2FeBrkuYAr9C7e7OaNa2oBUGjgP8iu21WJTnTp4CTyW7/dF69Y83MrDGFpbFN07i+T+dZK6dGxB/rH2VmZo3qs3zkZmZWjKKmH3Z3b7+IiJ8VUa+Z2YKoqDHy73VRPJhsEcbwiFiy5ZWamS2gCh9aSdPDDicL4lcDp0XEtEIrNTNbgBR2q7c01fAIsjm7FwMb1U5HNDOz3itqjPxUsmRJo4FPRsSsIuoxM7Pixsg7gNnAXDovfxbZxc6hLa/UzGwB5emHZmYl5+XyZmYl50BuZlZyDuRmZiXnQG5mVnIO5GZmJff/AdOtaKz3MTI2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"SketchFix\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=1)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "\n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['SketchFix-gen','NOT-SketchFix-gen']); ax.yaxis.set_ticklabels(['SketchFix-gen' , 'NOT-SketchFix-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "    auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "    print('ROC AUC: %f' % auc)\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cohens kappa: -0.071429\n",
      "ROC AUC: 0.285714\n",
      "Accuracy: 0.866667\n",
      "Precision: 0.000000\n",
      "Recall: 0.000000\n",
      "F1 score: 0.000000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEWCAYAAAB7QRxFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcZNP9//HXexBkjGWsGdvYkyDEGiRCxFgyliRiiZ2YkEgkxDeECL7kS3xJ7Iw1lhFEBkEwkV++llhmyWCIEPss1mEY60z35/fHPU1Nq+6+VV23q27P++lxH1N1761zTne1T5363HPPUURgZmblNaDZDTAzs95xIDczKzkHcjOzknMgNzMrOQdyM7OScyA3Mys5B3LrNUkLSvqzpBmSru9FOXtKurORbWsGSX+RtG+z22FzDwfyuYik70oaJ2mmpGkp4Hy5AUXvAiwNLB4R36m3kIi4OiKGNaA9c5C0haSQNLrT/nXS/r/nLOd4SVf1dF5EbBcRv6+zuWY1cyCfS0g6HPgd8GuyoLsCcB6wUwOKXxF4MiJmN6CsorwKbCJp8Yp9+wJPNqoCZfz/lPU5/9HNBSQtApwI/DAi/hQR70TErIj4c0Qcmc6ZX9LvJE1N2+8kzZ+ObSFpsqQjJL2SevP7p2MnAMcBu6We/oGde66Shqae77zp+X6SnpH0tqRnJe1Zsf/eitdtKmlsStmMlbRpxbG/S/pvSfelcu6UtEQ3v4YPgRuB3dPr5wF2A67u9Ls6U9KLkt6SNF7SV9L+bYFfVPycD1e042RJ9wHvAiunfd9Lx8+XdENF+adKukuScr+BZj1wIJ87bAIsAIzu5pxjgC8B6wLrABsBx1YcXwZYBFgWOBA4V9JiEfErsl7+tRGxUERc0l1DJA0EzgK2i4hBwKbAxCrnDQZuTecuDpwB3NqpR/1dYH9gKeBTwM+6qxu4AtgnPd4GmARM7XTOWLLfwWBgFHC9pAUi4vZOP+c6Fa/ZGxgBDAKe71TeEcDa6UPqK2S/u33Dc2NYAzmQzx0WB17rIfWxJ3BiRLwSEa8CJ5AFqA6z0vFZEXEbMBNYo872tANrSVowIqZFxGNVzvkG8FREXBkRsyPiGuAJYIeKcy6LiCcj4j3gOrIA3KWI+AcwWNIaZAH9iirnXBURr6c6Twfmp+ef8/KIeCy9Zlan8t4l+z2eAVwF/CgiJvdQnllNHMjnDq8DS3SkNrowhDl7k8+nfR+V0emD4F1goVobEhHvkKU0DgamSbpV0mdztKejTctWPH+pjvZcCRwKbEmVbyiSfibpXymd8ybZt5DuUjYAL3Z3MCIeBJ4BRPaBY9ZQDuRzh/uBD4CduzlnKtlFyw4r8Mm0Q17vAJ+ueL5M5cGIuCMitgY+Q9bLvihHezraNKXONnW4EvgBcFvqLX8kpT7+C9gVWCwiFgVmkAVggK7SId2mSST9kKxnPzWVb9ZQDuRzgYiYQXZB8lxJO0v6tKT5JG0n6TfptGuAYyUtmS4aHkeWCqjHRGBzSSukC61HdxyQtLSknVKu/AOyFE17lTJuA1ZPQybnlbQb8HngljrbBEBEPAt8leyaQGeDgNlkI1zmlXQcsHDF8ZeBobWMTJG0OnASsBdZiuW/JHWbAjKrlQP5XCLlew8nu4D5Klk64FCykRyQBZtxwCPAo8CEtK+eusYA16ayxjNn8B2Q2jEVmE4WVA+pUsbrwHCyi4Wvk/Vkh0fEa/W0qVPZ90ZEtW8bdwC3kw1JfB54nznTJh03O70uaUJP9aRU1lXAqRHxcEQ8RTby5cqOEUFmjSBfPDczKzf3yM3MSs6B3Mys5BzIzcxKzoHczKzkurtBpKlmvfaMr8LaJ6yyeiPm+LL+5oXpj/Z67ppaYs58S6zcUnPluEduZlZyLdsjNzPrU+1tzW5B3RzIzcwA2lp5Ov3uOZCbmQER1WaKKAcHcjMzgPbyBnJf7DQzA4j2/FsPJF2aVtOaVLHvNElPSHpE0mhJi3bx2uckPSppoqRxeZruQG5mBtnFzrxbzy4Htu20bwywVkR8gWxitqM7v6jClhGxbkRskKcyB3IzM2hojzwi7iab3bNy350Vi7M8ACzXqKY7kJuZAdE2O/cmaYSkcRXbiBqrOwD4S1dNAe5Mi3/nKtcXO83MoKaLnRExEhhZTzWSjiFbwOTqLk75ckRMkbQUMEbSE6mH3yX3yM3MoKGpla5I2o9swZQ9o4vFICJiSvr3FbJ1ZTfqqVwHcjMzaPTFzk+QtC3ZSlc7dl4vtuKcgZIGdTwGhgGTqp1byYHczAwaPfzwGrJFz9eQNFnSgcA5ZOvCjklDCy9I5w6RdFt66dLAvZIeBh4Cbo2I23uqzzlyMzNo6C36EbFHld2XdHHuVGD79PgZYJ1a63MgNzODUt/Z6UBuZgZEePZDM7Ny86RZZmYl59SKmVnJuUduZlZybbOa3YK6OZCbmYFTK2ZmpefUiplZyblHbmZWcg7kZmblFr7YaWZWcs6Rm5mVnFMrZmYl5x65mVnJuUduZlZy7pGbmZXc7MYtLNHXHMjNzMA9cjOz0nOO3Mys5NwjNzMrOffIzcxKzj1yM7OS86gVM7OSi2h2C+rmQG5mBs6Rm5mVngO5mVnJ+WKnmVnJtbU1uwV1cyA3MwOnVszMSs+B3Mys5JwjNzMrt2gv7zjyAc1ugJlZS2hvz7/1QNKlkl6RNKli32BJYyQ9lf5drIvX7pvOeUrSvnma7kBuZgbZqJW8W88uB7bttO8o4K6IWA24Kz2fg6TBwK+AjYGNgF91FfArOZCbmUFDe+QRcTcwvdPunYDfp8e/B3au8tJtgDERMT0i3gDG8MkPhE9wjtzMDPpi1MrSETEtPX4JWLrKOcsCL1Y8n5z2dcuBvMUc++szuPu+hxi82KLceNUFAJw98gr+du/9DNAABi+2CCcfcwRLLbl4k1tqzXLa2Sey1bDNef216Wy92bea3Zz+o4ZJsySNAEZU7BoZESPzVxUhqWFXV51aaTE7b781F5xx0hz79t/z24y+4nxu+P25fHWzjTn/slFNap21gutH3cQ+3zmk2c3of2pIrUTEyIjYoGLLE8RflvQZgPTvK1XOmQIsX/F8ubSvWw7kLWaDdddmkYUHzbFvoYEDP3r83nvvI/V1q6yVPHT/eN58Y0azm9H/tEf+rT43Ax2jUPYFbqpyzh3AMEmLpYucw9K+bhWaWpG0JHAQMLSyrog4oMh6+6MzL7ycm2+/i0EDB3Lp2ac0uzlm/U8D51qRdA2wBbCEpMlkI1FOAa6TdCDwPLBrOncD4OCI+F5ETJf038DYVNSJEdH5ouknFN0jvwlYBPgrcGvFVpWkEZLGSRp38RXXFNy0cjns+/tx1+gr+cawLRl1w5+b3Ryzfifa23NvPZYVsUdEfCYi5ouI5SLikoh4PSK2iojVIuLrHQE6IsZFxPcqXntpRKyatsvytL3oi52fjoif5z055ZlGAsx67Zny3mZVoOHDtuSQnx3Hod/bu9lNMetffGdnl26RtH3BdfR7z7/48bWOv91zPyutuFwTW2PWT0V7/q3FFN0jPwz4haQPgQ8BkY28WbjgekvryF+dwth/PsKbb77FVjvvxQ8O3Jt77h/Lcy9MRgPEkGWW4rgjf9TsZloTnX3RqWyy2YYstviiPDjpr5xxyrlce9XoZjer/ErcI1e06IKjTq1YNausvlOzm2At6IXpj/Z6LNc7x+2eO+YMPPEPLTV2rNDUijJ7Sfpler68pI2KrNPMrC4lTq0UnSM/D9gE+G56PhM4t+A6zcxqV/w48sIUnSPfOCLWk/RPgIh4Q9KnCq7TzKxmeYYVtqqiA/ksSfMAAR/dIFTe35aZ9V8t2NPOq+hAfhYwGlhK0snALsCxBddpZlY7B/LqIuJqSeOBrciGHu4cEf8qsk4zs7o08Bb9vlb0XCuDyWb4uqZi33wRMavIes3MalXmNTuLTq1MIJuS8Q2yHvmiwEuSXgYOiojxBddvZpZPiQN50cMPxwDbR8QSEbE4sB1wC/ADsqGJZmatoYFLvfW1ogP5lyLio7l0I+JOYJOIeACYv+C6zczy8zjyLk2T9HPgD+n5bmSrZMyDhyGaWStpwQCdV9GB/LtkE6rfmJ7fm/bNQ5pU3cysFURbefuWRQ8/fA34EWRr1FWsIA3wnyLrNjOrSYl75H25ZmeXKwOZmTVbtEfurdUUnVqp1FLTPpqZzaEFA3RefRnIL+rDuszMalPeFHnh85F/veNxRJyX9u1bZJ1mZvWI2e25t1ZTdI78OEnnSxooaWlJfwZ2KLhOM7PatdewtZiiA/lXgaeBiWRDD0dFxC4F12lmVrMyX+wsOpAvBmxEFsw/AFaU5IueZtZ63CPv0gPA7RGxLbAhMAS4r+A6zcxqVuYeedGjVr4eES8ARMR7wI8lbV5wnWZmtWvBnnZehQRySZ+NiCeAJSQt0enwzCLqNDPrjZjd7BbUr6ge+RHAQcDpVY4F8LWC6jUzq0v05x65pG8BYyLibUlHAesBv46IiV29JiIOSv9u2bCWmpkVqcSBPM/FzuNTEN8U2B64GriguxdI+nXF461710Qzs+JFe/6t1eQJ5B0rkg4HLoyIm+h5UYhtKx6fWk/DzMz6UpkDeZ4c+TRJ55IF5w0kfYq+nTXRzKxw0VbeW1zyBPJdyVIqZ0fEG5KGAEf18JqlJB1ONuNhx+OPRMQZdbXWzKwgrdjTzqvLnrWkhSUtnM65HZians+k55t6LgIGAQtVPK7czMxaSrQr99YdSWtImlixvSXpJ53O2ULSjIpzjutN27vrkT9GNlSwstUdzwNYoasXRsQJvWmUmVlfa1SPPCL+DawLkNYnngKMrnLqPRExvBF1dhnII2L53hYu6TfAScB7ZL36LwA/jYirelu2mVkjRRSSI98KeDoini+i8A65LlpK2l3SL9Lj5SStn7P8YRHxFtmIl+eAVYEj62momVmRahm1ImmEpHEV24guit0duKaLY5tIeljSXySt2Zu257kh6BxgPmBz4NfAu2TjyDfMUf586d9vANdHxAxPfmhmrai9hlErETESGNndOWmE347A0VUOTwBWjIiZkrYHbgRWy9/aOeXpkW8aEd8H3geIiOnAp3KWf7OkJ4D1gbskLdlRjplZK2nUxc4K2wETIuLlT9QV8VZEzEyPbwPmqzIvVW55hh/OkjSA7AInkhYn/82sE4DzyZL9R5P16g/v9hVmZk1QQ4DOaw+6SKtIWgZ4OSJC0kZknerX660oT4/8XOAGYElJJ5Ct9JP3bs1fpmlsNwG+DpwJeAy5mbWciPxbTyQNBLYG/lSx72BJB6enuwCTJD0MnAXsHpGn5Op67JFHxBWSxpMFYoDvRMSknOV33N7/DWBkRNwq6aQ62mlmVqhG9sgj4h1g8U77Lqh4fA5wTqPqyzuN7TzALLL0Si2350+RdCHZJ9Opkuav8fVmZn2ioOGHfaLHoCrpGLI8zxBgOWCUpGpXYavZFbgD2CYi3gQG4+GHZtaC2tqUe2s1eXrk+wBfjIh3ASSdDPwT+J+eXphe86eK59OAafU11cysOGXukeea/bDTefPiYGxm/UwBo1b6TJeBXNJvyXLi04HHJN2Rng8DxvZN88zM+kb9Y0aar7seecfIlMeAWyv2P1Bcc8zMmqNf9sgj4pK+bIiZWTO1tZd3QF2euVZWAU4GPg8s0LE/IlYvsF1mZn2qzKmVPB9BlwOXkc1Dvh1wHXBtgW0yM+tz7aHcW6vJE8g/HRF3AETE0xFxLFlANzPrNyKUe2s1eYYffpAmzXo6zRMwBS/XZmb9TJlTK3kC+U+BgcCPyXLliwAHFNkogFVW36noKqyEps6c3uwmWD/ViimTvPJMmvVgevg2sHexzTEza45+OWpF0mjSHOTVRMS3CmmRmVkTlDiz0m2PvGFTLJqZtbp+mVqJiLv6siFmZs3UiqNR8so7H7mZWb+Wd/3KVuRAbmYGBHNBj1zS/BHxQZGNMTNrltklTq3kWSFoI0mPAk+l5+tIOrvwlpmZ9aFAubdWk2fg5FnAcOB1gIh4GNiyyEaZmfW19hq2VpMntTIgIp6X5vgUaiuoPWZmTdGKPe288gTyFyVtBISkeYAfAU8W2ywzs77Vij3tvPIE8kPI0isrAC8Df037zMz6jbb+3COPiFeA3fugLWZmTVPild5yrRB0EVWmIYiIEYW0yMysCdr7c4+cLJXSYQHgm8CLxTTHzKw5+uukWQBExBzLukm6Eri3sBaZmTVBf7/Y2dlKwNKNboiZWTO1qx+nViS9wcffOgYA04GjimyUmVlfK/PNMd0GcmV3Aa1Dtk4nQHtEmVe2MzOrrsyjVrq9RT8F7dsioi1tDuJm1i+1o9xbq8kz18pESV8svCVmZk0UNWytprs1O+eNiNnAF4Gxkp4G3gFE1llfr4/aaGZWuEamViQ9R7ZgfRswOyI26HRcwJnA9sC7wH4RMaHe+rrLkT8ErAfsWG/hZmZlUcDwwy0j4rUujm0HrJa2jYHz07916S6QCyAinq63cDOzsmjr29T3TsAV6brjA5IWlfSZiJhWT2HdBfIlJR3e1cGIOKOeCs3MWlEtPXJJI4DKaUpGRsTIiucB3CkpgAs7HQNYljnvkJ+c9jU8kM8DLAQteInWzKzBagnkKTB3Ds6VvhwRUyQtBYyR9ERE3N27Fnatu0A+LSJOLKpiM7NW0sglOyNiSvr3FUmjgY2AykA+BVi+4vlyfHy/Ts26G37onriZzTUatdSbpIGSBnU8BoYBkzqddjOwjzJfAmbUmx+H7nvkW9VbqJlZ2TTwFv2lgdFpecx5gVERcbukgwEi4gLgNrKhh/8hG364f28q7DKQR8T03hRsZlYmjRpHHhHPkE1t0nn/BRWPA/hhY2qsb/ZDM7N+Z26bxtbMrN9xIDczK7lWnEMlLwdyMzPKPY2tA7mZGf14YQkzs7lFe4mTKw7kZmb4YqeZWemVtz/uQG5mBrhHbmZWerNV3j65A7mZGU6tmJmVnlMrZmYl5+GHZmYlV94w7kBuZgY4tWJmVnptJe6TO5CbmeEeuZlZ6YV75GZm5VbmHvmAZjfAunba2Scy4d9/Z8x9f2p2U6zFbDNsCx6bdDdPPH4v/3Vkw5Z+nKu1E7m3VuNA3sKuH3UT+3znkGY3w1rMgAEDOOvMkxm+w16svc6W7Lbbznzuc6s1u1mlFzVsrcaBvIU9dP943nxjRrObYS1mow2/yNNPP8ezz77ArFmzuO66m9hxh22a3azSm03k3lqNA7lZyQxZdhlenDz1o+eTp0xjyJBlmtii/iFq+K/VFHqxU9L8wLeBoZV1RcSJXZw/AhgBsNinh7DQ/IOLbJ6Z2UfKfLGz6FErNwEzgPHABz2dHBEjgZEAKwxeu/U+9sxawNQpL7H8ckM+er7csp9h6tSXmtii/qEVe9p5FR3Il4uIbQuuw2yuMnbcRFZddSWGDl2eKVNeYtddd2LvfTxypbfK3CMvOkf+D0lrF1xHv3X2Rady4x1XsfKqQ3lw0l/Zba9vNrtJ1gLa2to47CfHctuto5j0yN/54x//zOOPP9nsZpVeW0TurdUoCmyUpMeBVYFnyVIrAiIivtDTa51asWqmzpze7CZYC5r94RT1tozvrvjN3DFn1POje11fIxWdWtmu4PLNzBqizDnyQlMrEfE8sDzwtfT43aLrNDOrR3sNW6spevjhr4ANgDWAy4D5gKuAzYqs18ysVq14631eRadWvgl8EZgAEBFTJQ0quE4zs5qVObVSdCD/MCJCUgBIGlhwfWZmdWnF0Sh5FZ2vvk7ShcCikg4C/gpcVHCdZmY1a9Tsh5KWl/T/JD0u6TFJh1U5ZwtJMyRNTNtxvWl7oT3yiPhfSVsDb5HlyY+LiDFF1mlmVo8GXsScDRwRERNSKnm8pDER8Xin8+6JiOGNqLDwhSVS4HbwNrOW1qgceURMA6alx29L+hewLNA5kDdMoakVSW9LeqvT9qKk0ZJWLrJuM7Na1JJakTRC0riKbUS1MiUNJRvw8WCVw5tIeljSXySt2Zu2F90j/x0wGRhFdlfn7sAqZKNYLgW2KLh+M7NcarnLvXKCv65IWgi4AfhJRLzV6fAEYMWImClpe+BGoO7VQYq+2LljRFwYEW9HxFvph98mIq4FFiu4bjOz3NqI3FtPJM1HFsSvjohPrNWY4uHM9Pg2YD5JS9Tb9qID+buSdpU0IG27Au+nY+Ud62Nm/U4DR60IuAT4V0Sc0cU5y6TzkLQRWSx+vd62F51a2RM4EzgvPb8f2EvSgsChBddtZpZbAycQ3AzYG3hU0sS07xfACqmeC4BdgEMkzQbeA3aPXjSg6OGHzwA7dHH43iLrNjOrRaNu0Y+Ie8muCXZ3zjnAOQ2pkD6cwErShL6qy8ysVl6zM5+Wmr/XzKxSmW/R78tAfmsf1mVmVpMyz35Y9A1Bn+94HBHHpn1bFFmnmVk9GjVqpRn6YtKsnyuzoKSzgf8puE4zs5pFRO6t1RQdyDcmWyHoH8BYYCpeVMLMWlCZe+RF58hnkY2RXBBYAHg2IlpxpSQzm8u14miUvIrukY8lC+QbAl8B9pB0fcF1mpnVrC3ac2+tpuge+YERMS49ngbsJGnvgus0M6tZK+a+8yokkEtaOM329YykwZ0OexiimbWcVsx951VUj3wUMBwYTzY5VuXNQAF4LnIzayllzpEXEsg7li+KiJWKKN/MrNHaS5xaKeRip6RDKx73auULM7O+UOa5VooatXJAxeMrC6rDzKxhPGqle54sy8xaXplTK0UF8kUlfZOsx7+wpG9VHqy29JGZWTO1Ysokr6IC+f8BO6bHdzPn4hIBOJCbWUtxj7yTiNi/iHLNzIpS5h550dPYHiZp4TT74cWSJkgaVmSdZmb1aIu23FurKXqulQPSHZ7DgMXJFiQ9peA6zcxqVuZpbIsetdIxYmV74IqIeEySR7GYWcvxLfpdGy/pTmAl4GhJg4DWG4RpZnO9Vuxp51X47IfAusAzEfFumkDLF0LNrOWUedRK0TnyTYB/R8SbkvYCjgVmFFynmVnNfIt+184H3pW0DnAE8DRwRcF1mpnVrMy36BcdyGdHlnjaCTgnIs4FBhVcp5lZzTxqpWtvSzoa2AvYXNIAYL6C6zQzq5lz5F3bDfiAbMm3l4DlgNMKrtPMrGbukXchBe8zACQNj4hbcI7czFpQmceRF90jr3RiH9ZlZlYT98jz8R2dZtayWnE0Sl5FLfX2pSq7v19EXWZmjdAekXtrNUWlVs7rvCMiHiqoLjOzXitzaqUvc+RmZi2rkXd2StpW0r8l/UfSUVWOzy/p2nT8QUlDe9P2onLkK0u6uauDEbFjV8fMzJqhUT1tSfMA5wJbA5OBsZJujojHK047EHgjIlaVtDtwKtlw7boUFchfBU4vqGwzs4ZrYO57I+A/EfEMgKQ/kN3dXhnIdwKOT4//CJwjSVHnp0lRgfztiPi/3hTwwvRHPcolkTQiIkY2ux3WWvx30VizP5ySO+ZIGgGMqNg1suK9WBZ4seLYZGDjTkV8dE5EzJY0g2zxnddqbTcUlyN/rqBy51Yjej7F5kL+u2iSiBgZERtUbE39QC1q8eVvSVoK+CGwZtr9GHBeRLxcRJ1mZi1iCrB8xfPl0r5q50yWNC+wCPB6vRUWNY58M2BsenoFH9+W/2A6ZmbWX40FVpO0kqRPAbsDnQd/3Azsmx7vAvyt3vw4FJcjPx3YOSL+WbHvZkmjgQv5ZL7Iuuc8qFXjv4sWlHLehwJ3APMAl6b1ik8ExkXEzcAlwJWS/gNMJwv2dVMRg9slPR4Rn6/1mJmZ1a6oi52StFiVnYMLrNPMbK5UVFD9LXCnpK9KGpS2LYC/pGNmZtYghQTyNBTnBOC/yYYiPkc2je1JEXFhEXU2mqSQdHrF859JOr7i+QhJT6TtIUlfTvtHS5qYbr2dkR5PlLRpL9pyuaRnK8r6cdp/m6RFe/FjWouQdLykKRXv8Slp/8WSnIq0bhWSI+8PJL0PTAM2jIjXJP0MWCgijpc0nOyDapt0bD3gRmCjtJgG6RvIzyJieAPacjlwS0T8sbdlWWtKnYSZEfG/zW6LlU9Rww+P62b7ZRF1FmA22aiAn1Y59nPgyIh4DSAiJgC/Jxs3n4ukz0i6O/W+Jkn6iqR5Uu97kqRHJVWru7KM5yQtIWlDSY9IWkDSQEmPSVqryvmrSHoglX2SpJkVx46UNDaVc0LaN1TSvyRdlMq8U9KCeX9Gg/R+3Crp4fS+7ibpFEmPp991t4Fb0t8lbSBpRUlPpfd7gKR7JA2rcv6Sksak9+tiSc9LWiId2yt9e5wo6UJlc4Igaaakk1MbH5C0dDG/DStKUTnyd6pskE0U8/OC6izCucCekhbptH9NYHynfeP4+OanPL4L3BER6wLrABOBdYFlI2KtiFgbuKzi/NMqvnavXVlQRIwlG5d6EvAb4KqImFSlzjOBM1PZkzt2poCwGtkcEesC60vaPB1eDTg3ItYE3gS+XcPPaLAtMDUi1omItYAHgG8Ca0bEF8jesw4/rXiPt6ksJCKeJ5tY6XzgCODxiLizSn2/IhuTvCbZHB4rAEj6HNmkTJulv7k2YM/0moHAAxGxDnA3cFAjfnDrO0Xd2VmZWx4EHAbsD/yBEk2mFRFvSboC+DHwXoOLHwtcKmk+4MaImCjpGbKZI88GbgUq/0c9sofUyompzPdTe6vZBNg5PR4FdPQGh6WtY9z/QmQB/AXg2YiYmPaPB4bm+/EseRQ4XdKpwC3A/WTv0SWSbkn7Ovy2u9RKRFws6TvAwWQfuNV8meyDgoi4XdIbaf9WwPpkM/EBLAi8ko59WNGO8WSz9lmJFDYUUNJgSScBj5B9YKwXET+PiFd6eGmr+R3ZN4mBFfseJ/ufotL6ZNMQVCVp44re1o4RcTewOdmtupdL2ici3iDrnf+d7H/Wi2to5+JkAXgQsECq8+SOOnt4rYD/iYh107ZqRFySjn1QcV4bfbs8YOlFxJPAemQB/STgF2TffP4IDAduz1uWpE+T3e4N2XuNpB9W/F0N6e7lwO8r3uM1IuL4dGxWxV2Ffo9LqKgc+WlhiBp6AAAE+klEQVRkvcO3gbUj4vgUpEonIqYD15EF8w6/AU6VtDiApHWB/aiyMlJFOQ9W/E90s6QVgZcj4iKygL1eymUOiIgbgGPJAkBeFwK/BK4m+wpORBzTUWc65wE+To1U3kl2B3CApI7gsKyyuXKsl1JwfTcirgJOI/vwXiQibiO7/rJODcWdSvb+HgdcBBAR51b8XU0F7gN2TXUPAzru57gL2KXjfU0drRV7/QNaSyjqk/cIsp7cscAx6ascZL2CiIiFC6q3KKcDh3Y8SYF4WeAfkoLsA2uviJhWQ5lbAEdKmgXMBPYhm9ryMkkdH7BH5ylI0j5kvapR6QLWPyR9LSL+1unUnwBXSTqGrCc4I/08d6Yc6v3pvZoJ7EXWO7PeWZvs+kY7MAs4HLhF0gJk/z8cnqcQSV8FNiTLcbdJ+rak/SPisk6nngBcI2lvsjTOS2TTSr8m6Viy+zsGpLb8EHi+AT+jNZmHH85F0lfz9yIilK1KskdE7NTsdlnjSJofaEvzfWwCnF/xjcz6KefC5i7rk1YiIRuBckCT22ONtwJwXep1f4hHoMwV3CM3Mys5T2BlZlZyDuRmZiXnQG5mVnIO5PYJktr08Rww16fRLvWWtUW6gxFJO0o6qptzF5X0gzrqOF7ZpGa59nc653JJu9RQ11BJ1aY/MGsaB3Kr5r10g8laZCMfDq48qEzNfzsRcXNEnNLNKYsCNQdys7mdA7n15B5g1dQT/Xeae2YSsLykYZLulzQh9dw77gzdVtk87ROAb3UUJGk/Seekx0srm7v94bRtCpwCrJK+DZyWzvvErIxp/zGSnpR0L7BGTz+EpINSOQ9LuqHTt4yvSxqXyhuezp9H0mkVdX+/Splr6uPZBB+RtFrtv16z3nMgty5JmhfYjmyeEMgm0jovzaz3Dtmdu1+PiPXIZn88PN2xeBGwA9m49WW6KP4s4P/SjHvrkc1TcxTwdPo2cKS6mJVR0vpkUwysC2xPdsdjT/4UERum+v7FnFMuDE11fAO4IP0MBwIzImLDVP5BklbqVObBZLNJrgtsQMWMkmZ9yTcEWTULVky0dQ/Zit9DgOcj4oG0/0vA54H70m39nyK7JfyzZDMmPgUg6SpgRJU6vkY2LQER0QbM0CfXee1qVsZBwOiIeDfVcXOOn2ktZZO4LZrKuaPi2HUR0Q48pWwGys+mer9QkT9fJNX9ZMXr7iebgmI5sg+Kp3K0w6zhHMitmvc639adgvU7lbuAMRGxR6fzGnk7eMesjHMsDyjpJ3WUdTmwc0Q8LGk/srluOnS+Ky5S3T+KiMqAj6ShH52UzW3zIFlP/jZJ368yv41Z4ZxasXo9AGwmaVX4aCWc1YEngKGSVknn7dHF6+8CDkmvnUfZ4h1vk/W2O3Q1K+PdwM6SFlQ23/0OOdo7CJimbP73PTsd+46yVXdWAVYG/p3qPiSdj6TVJVVOZYyklYFnIuIs4CbgCznaYdZw7pFbXSLi1dSzvSZN1ARwbEQ8KWkEcKukd8lSM4OqFHEYMFLSgWSzLB4SEfdLui8N7/tLypN/YlbGiJgg6VrgYbLFEcbmaPIvgQeBV9O/lW16AXgIWBg4OCLel3QxWe58Qpqb5lU+XpSjw67A3spmsHwJ+HWOdpg1nOdaMTMrOadWzMxKzoHczKzkHMjNzErOgdzMrOQcyM3MSs6B3Mys5BzIzcxK7v8DqUC5pvVb2tkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "    tool=\"ssFix\"\n",
    "    y = dummy[tool]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=11)\n",
    "\n",
    "    svclassifier = SVC(kernel='linear')\n",
    "    svclassifier.probability=True\n",
    "    svclassifier.fit(X_train, y_train)\n",
    "    y_pred = svclassifier.predict(X_test)\n",
    "    y_prob = svclassifier.predict_proba(X_test)\n",
    "\n",
    "#     print(confusion_matrix(y_test, y_pred))\n",
    "#     print(classification_report(y_test, y_pred))\n",
    "\n",
    "\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    ax= plt.subplot()\n",
    "    sns.heatmap(cm, annot=True, ax = ax); #annot=True to annotate cells\n",
    "\n",
    "    # labels, title and ticks\n",
    "    ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \n",
    "    ax.set_title('Confusion Matrix'); \n",
    "    ax.xaxis.set_ticklabels(['NOT-ssFix-gen', 'ssFix-gen']); ax.yaxis.set_ticklabels(['ssFix-gen' , 'NOT-ssFix-gen']);\n",
    "    \n",
    "    kappa = cohen_kappa_score(y_test, y_pred)\n",
    "    print('Cohens kappa: %f' % kappa)\n",
    "    # ROC AUC\n",
    "    auc = roc_auc_score(y_test, y_prob[:,0])\n",
    "    print('ROC AUC: %f' % auc)\n",
    "    # accuracy: (tp + tn) / (p + n)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    print('Accuracy: %f' % accuracy)\n",
    "    # precision tp / (tp + fp)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    print('Precision: %f' % precision)\n",
    "    # recall: tp / (tp + fn)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    print('Recall: %f' % recall)\n",
    "    # f1: 2 tp / (2 tp + fp + fn)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    print('F1 score: %f' % f1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}