Gridsearch #4 SVM Linear.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pylab as pl\n",
    "from sklearn.model_selection import train_test_split\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.metrics import plot_confusion_matrix\n",
    "\n",
    "from sklearn import datasets\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.000210523684764536 0.005160091115494809 0.1361031518624642 ... 0 0 0]\n",
      " [0.06578947122576177 0.013897588526412541 0.14469914040114612 ... 0 0 0]\n",
      " [0.005263155277008304 0.010694145453587434 0.11891117478510028 ... 0 1 0]\n",
      " ...\n",
      " [0.05263157645429362 0.859810398981424 0.1346704871060172 ... 0 0 0]\n",
      " [0.000988368423653601 0.005792679656356735 0.1017191977077364 ... 1 0 0]\n",
      " [0.17105262939750693 0.057828596847734885 0.13753581661891118 ... 0 0 0]] ['Bad' 'Bad' 'Bad' ... 'Average' 'Bad' 'Average']\n",
      "4800 4800 1200 1200\n"
     ]
    }
   ],
   "source": [
    "arr2= np.load('data_array_formatted.npy', allow_pickle= True)\n",
    "X= arr2[:,0:-1]\n",
    "Y= arr2[:,-1]\n",
    "\n",
    "\n",
    "### Split arrays or matrices into random train and test subsets - 20% for Test, 80% to train\n",
    "X_train, X_test, y_train, y_test= train_test_split(X, Y, test_size=0.20,  random_state = 200)\n",
    "\n",
    "print(X_train,y_train)\n",
    "print(len(X_train),len(y_train),len(X_test),len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM Linear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.84 s, sys: 0 ns, total: 5.84 s\n",
      "Wall time: 5.83 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score=nan,\n",
       "             estimator=LinearSVC(C=1.0, class_weight=None, dual=True,\n",
       "                                 fit_intercept=True, intercept_scaling=1,\n",
       "                                 loss='squared_hinge', max_iter=1000,\n",
       "                                 multi_class='ovr', penalty='l2',\n",
       "                                 random_state=None, tol=0.0001, verbose=0),\n",
       "             iid='deprecated', n_jobs=None,\n",
       "             param_grid=[{'C': [0.1, 1, 10, 100, 1000]}],\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1_macro', verbose=0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "tuned_parameters = [{'C': [0.1,1, 10, 100, 1000]}]\n",
    "clf = GridSearchCV(LinearSVC(), tuned_parameters, scoring='f1_macro')\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters set found on development set:\n",
      " {'C': 10}\n",
      "\n",
      " Best Score 0.5799626573453999\n",
      "\n",
      " Grid scores on development set:\n",
      "0.534 (+/-0.021) for {'C': 0.1}\n",
      "0.577 (+/-0.016) for {'C': 1}\n",
      "0.580 (+/-0.014) for {'C': 10}\n",
      "0.577 (+/-0.021) for {'C': 100}\n",
      "0.479 (+/-0.063) for {'C': 1000}\n",
      "Detailed classification report:\n",
      "The scores are computed on the full evaluation set. \n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Average       0.64      0.30      0.41       405\n",
      "         Bad       0.59      0.78      0.67       392\n",
      "        Good       0.60      0.72      0.65       403\n",
      "\n",
      "    accuracy                           0.60      1200\n",
      "   macro avg       0.61      0.60      0.58      1200\n",
      "weighted avg       0.61      0.60      0.58      1200\n",
      "\n",
      "\n",
      "Testing score:  0.5772180146980803\n",
      "\n",
      "Training Score:  0.582978291944246\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEGCAYAAAApAy29AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3wVxdrA8d+TBEgghN67FBGQfhUEKRYELBQbiBRFUMT2ehXBcvXqVbGiYuFSpKiAeBXBCgIqooD0XlWaIKETIASSPO8fu4EAyclJOWXx+frZD+fMzpmZXY4Pc2ZnZ0VVMcYYE3gRoW6AMcb8XVjANcaYILGAa4wxQWIB1xhjgsQCrjHGBElUqBsQruKKFddS5SqFuhlhK8Vmt2TpZKqdI18O/vUnRw/tl9yUERlXRTU50a+8mrhnhqq2z019uWUBNxOlylXipYnfhLoZYetg0slQNyHsxR+xc+TLO/d2yXUZmpxIgQtv8Svv8eXvlMx1hblkQwrGGA8TkAj/tqxKEokWkV9FZIWIrBGRf7vp1URkoYhsEpGPRSS/m17Afb/Z3V81qzos4BpjvEuAiEj/tqwlAVeoagOgIdBeRJoBLwHDVLUmcADo6+bvCxxQ1RrAMDefTxZwjTHeJuLflgV1HHHf5nM3Ba4A/uemjwc6u687ue9x918p4rsiC7jGGA/L1pBCSRFZnG7rf05pIpEishyIB74DfgMOqmqym2UHUMF9XQHYDuDuPwSU8NVau2hmjPE2P3qvrr2q2tRXBlVNARqKSFFgKnBRRtnSavaxL0PWwzXGeJeQZxfN0lPVg8APQDOgqIikdU4rAjvd1zuASgDu/iLAfl/lWsA1xniYn+O3fvSCRaSU27NFRGKAq4B1wPfATW623sA09/V09z3u/jmaxfKLNqRgjPE2/2Yg+KMcMF5EInE6o1NU9UsRWQtMFpH/AMuAMW7+McAHIrIZp2fbLasKLOAaYzxMsj1ckBlVXQk0yiD9d+CSDNKPAzdnpw4LuMYY7xKyc9Es5CzgGmO8LY96uMFgAdcY42F5N6QQDBZwjTHeJUBknl00CzgLuMYYb7MxXGOMCQYbUjDGmOCxHq4xxgSJ9XCNMSYI/LxtN1xYwDXGeFve3dobcBZwjTEeZhfNjDEmeGxIwRhjgiBtPVyPsIBrjPEwG1IwxpjgsYtmxhgTJDaGa4wxQSA2pGCMMcFjPVxjjAkOsYBrjDGB5zxhxwKuMcYEnggSYQH3b2/UmC9ZtmIzcXEFGfqf/ufs/3n+ar76ej4ABQrkp0+v9lSpXCZXdZ48mcx/R33BH1v/IjY2hvsGdKZUyaKsWvMHUz75nuTkFKKiIul2yxXUrVM1V3Xl1kcTvmH1qt8pXLggj//rjnP2r1y+ia++mIeIEBERwY23XEH1GhVzVefRo4mMHfUF+/cdoniJItzZ7wYKFopm0cK1zJq5EHD+Lm657WoqViydq7rywvSPZ7Jp7R8Uii3IPY/2PGf/ls3bmTLuC4oWjwOgdr0atGrXLFd1JicnM23SDHbtiCemYDQ39uxI0eJF+H3jVmZ/9TMpKSlERkZy1XWXU61mpVzVlVe81MMN6OU9EekiIioitQNZTzi6vGV9Bj2c+WPqS5UsyhODb+eF5/rR+YaWvD/+G7/L3rP3IM8P/fCc9B9/WkGhQtG89tIA2rf7Bx9P+R6AwrExPPzgzbz4n370v+s6/jtqevYPKI9d2rwe995/U6b7L6xdhcFP9mHwk33o0as9Ez+Y4XfZmzZs44NxX5+T/t23C6lVuwr/eq4ftWpX4bsZTpAtUbIIDz7cnSFP3cE1HZsz+cOZ2T+gAGjQtA639eviM0/lahXo//Dt9H/49mwF24P7DzHh3U/OSV++cA3RMdHcN+QOLm3VmNlfzQMgplAM3e68gXse6Umnbu2YNunb7B1MAImIX1s4CHQPtzswD+gGPJObgkQkUlVT8qJRwVD7wsrs2Xsw0/21ap7urdWoXp4D+w+fev/zL6uZOWsRyckpVL+gPH16tSciIut/G5cu3UiXzpcDcEnTi5jw4UxUlapVyp7KU7FCKU6eTOHkyWTy5QvdD5waNSuxb++hTPcXiM5/6vWJEyfPuBA9a+avLFuygeTkZOo3rMm117f0q85VKzfzgPuP4KXN6/LW65Pp1LU1F1SvcCpPtWrlOXggIZtHExhVqlfk4P7Mz5EvK5esY9G85aSkpFChclk6dL3Cr+/QhjW/0doN3HXq1+Tbqd+jqpSrcLrHX6psCZKTU0hOTiYqKvQ/ksMlmPojYD1cEYkFWgB9cQIuIvKxiHRMl2eciNwoIpEi8oqILBKRlSJyt7u/jYh8LyITgVVu2uciskRE1ohI/3Rl9RWRjSLyg4iMEpG33fRSIvKpW/YiEWkRqGPOqR/mrqD+xdUB+HPnXhb8upanHu/F88/eRUREBL/MX+NXOfsPJlDC/XkZGRlBwZgCHDmSeEaeRYvXU6VKmZAGW3+tWLaR554ew4i3P6NHr/YArFv7B3viD/DI4Nt57Ik+bN+2m82btvtVXsLhYxQpEgtAkSKxJCQcOyfP/J9XUqdetbw7iADbsXUX/33tQyaOmkr8X/sA2LN7P2uXb6TPfbfQ/+HbEYlg1dL1fpWXcOgocUULAxARGUF0TAESjx0/I8+6lZspW6FUWARb56qZn1sYCOQZ6wx8q6obRWS/iDQGJgO3Al+LSH7gSmAATlA+pKr/EJECwM8ikva77hKgnqr+4b6/U1X3i0gMsEhEPgUKAE8BjYEEYA6wws3/JjBMVeeJSGVgBnBRAI87W9au28Lcn1bw5OPOGN3atVvYsvUvnn52LAAnTiYTF1cQgDeG/489ew6SnJLCvn2HeeJfowG45up/0OryBqAZVJDui7bjzz18/Mn3DHqke0CPKa80aFSLBo1qsXnTdr6cPo/7H7qV9Wu3sH7tFl56fjwASUkn2RN/gBo1K/Hq0A9JTk4mKekkx44eZ+h/xgHQqUtrLqqbdRDduGEb839Zxf89clsgDyvPlKtYmgeeuJP8BfKzad0ffDLuCwYO7sOWTdvY9Wc8Y96cBMDJkykUio0BYMq4Lzi4/xApyakcOpjAyNedoalLWjai4SV10Qy/RKfF/7WPOV/Py3KoI1iE8Bku8EcgA2534A339WT3/VPAW25QbQ/MVdVEEWkH1BeRtEG9IkBN4ATwa7pgC/CAiKT9bVdy85UFflTV/QAi8glQy81zFVAn3V9KnIgUVtVzfje6Peb+ACXLVTh7d57btj2eMWO/5pGHb6VwrBNUFaXlZRdz681tz8n/kDvmuWfvQUaO/pInBt9+xv7ixQqzb/9hihePIyUllWOJScQWcv5H27//MG8O/5S7+11PmdLFAnxkeatGzUrs3XOII0eOocDV7S+lZauG5+R7xD0fmzZsY8H81fTs0/GM/YXjCnLo0BGKFInl0KEjFC5c8NS+P3fEM+mDbxlw/02nglO4KxBd4NTrmhdV45vP5nDsaCIK1G96EVd2PHeo5ZY+1wPOGO70yTPpde/NZ+yPKxLL4YMJxBUtTGpKKscTk4gpGA3A4YMJfDLuCzp1u4biJYsG7sCyyZ+hknARkJaKSAngCmC0iGwBHsXp2SYBPwDXuO8np30EuF9VG7pbNVVN6+EeTVduG5wA2lxVGwDLgGh8/2CIcPOnlV0ho2ALoKojVbWpqjaNK1oiJ4fut737DvHm259yd78bKFf2dF11L6rKosXrOXTYOewjRxLZ62OsM71GjWoy7+dVAPy6eB11LqqCiHD02HFefWMKt9zUhlphcmU5K3viD6Dq9La2b9tNSnIKhQrFcFGdqiz4ZTVJx08AcPBAAgmHj/oq6pSL69dgoTs8s3D+Gi6uXwNw/jEa/d9p9LzjWkqXKR6AowmMI4ePnjpHf277C1WIKRhNtRqVWL9yM0fdIZPEY8c5mO4agS+16lZnxeJ1AKxduYmqNSohIhxPPM6kMdO4omMLKlUrH5gDyiG7aAY3ARNU9e60BBH5EWiJE2TvApoCfdzdM4ABIjJHVU+KSC3gzwzKLQIcUNVj7syHtMuyvwLDRKQYzpDCjbhjvsBM4D7gFbcdDVV1eZ4daSbeGfE569Zv5ciRRB54eDhdO19OSkoqAFe2bczn0+Zx5Egi4z9wrvZGRkbw7NN3UqFCKW7q2pqXX52EqhIZGUnvntdQsmSRLOts3aohI0ZO55+PvUdsoWgG3tMZgO9mLWb37gN8Pn0en093rjoPeqQ7ReIKBejoszZ29Bds3ridI0cSeWrwe3S8vsWp89OyVUOWL9vIrwvWEBkZQb58UdzR73pEhIvqVGP3rv289vJHABQokI9ed15LYbI+lquvuZT3R01nwc8rKVY8jjv73wDAt1/9wtGjiUyZ9B3g9JgGPd4rQEfuv88+/Jqtv+3g2NHjvPHcaFq3a0aqe46aXFafdSs3sXj+SiIinHPU9fYOiAilypagTfvmfDTqM1Sd4+nQte2p6WO+NLqkLp9PmsHbL44lpmA0XW93fiUs+nkFB/Ye5KdZC/lpljO7o0e/rhRK9yshJMJofNYfkvYvZJ4WKvIDMFRVv02X9gDO2OkDwF/AdFW9w90XAfwHuB7n9O3BGQNuBDyiqte5+QoAnwMVgA1AKeAZVf3BHQ54BNgJrAP2q+oTIlISeMetOwpnGOOerI6hep0G+tJE/6dq/d0cTDoZ6iaEvfgjdo58eefeLvy5YVWuwmVUyQu06HUv+JV33/juS1S1aW7qy62A9HBVtU0GaW+le1virH2pwOPult4P7paWLwnokEm1E1V1pIhEAVNxerao6l6c4QtjzHkmLy+aiUglYALONaFUYKSqvikizwD9cDqCAI+r6tfuZ4bgXPRPAR5QVZ8TxsNgXkeeeUZErsIZ052J0xM2xpzn8vDW3mTgn6q6VEQKA0tE5Dt33zBVffWMekXq4Ex5rQuUB2aJSC1f9wucNwFXVR8JdRuMMUEmeXfjg6ruAna5rxNEZB3O8GVmOgGT3V/ef4jIZpxprPMz+4B35lMYY0wGsjFLoaSILE63nbvIyekyq+JcQ1roJt3n3pT1vntxHpxgnP6umx34DtAWcI0x3paNgLs3bdqnu43MpLxY4FPgIVU9DLwHVAca4vSAX0vLmsHHfc5COG+GFIwxfz95faeZiOTDCbYfqepnAKq6O93+UcCX7tsdODdfpamIM0sqU9bDNcZ4Wx6tpSBO5B4DrFPV19Oll0uXrQuw2n09HegmIgVEpBrOXa+/+qrDerjGGO+SPL21twXQE1glImk3Rz0OdBeRhjjDBVuAuwFUdY2ITAHW4sxwGJjVioYWcI0xnpaHsxTmkXFf+NzFlU9/5nngeX/rsIBrjPE2D93aawHXGONp4bIwjT8s4BpjPCucVgLzhwVcY4ynWcA1xpggscekG2NMkFgP1xhjgiEPF68JBgu4xhjPEsBD8dYCrjHGy2yWgjHGBE2EXTQzxpggEBtSMMaYoBCsh2uMMUFjPVxjjAkSu2hmjDHBYGO4xhgTHILk5QLkAWcB1xjjadbDNcaYILExXGOMCQYbwzXGmOBw1lLwTsS1gGuM8TQPxVsLuMYYb7M7zYwxJhhsPdzzQ1x0Pq6+sEyomxG2yl72YKibEPa2zh0W6iaEtSmF8ue6DFsP1xhjgsbWwzXGmKDxULy1gGuM8TCxi2bGGBMUNg/XGGOCyAKuMcYEiYfiLd5Z18wYYzIgIn5tfpRTSUS+F5F1IrJGRB5004uLyHcissn9s5ibLiLylohsFpGVItI4qzos4BpjvMtdvMafzQ/JwD9V9SKgGTBQROoAg4HZqloTmO2+B+gA1HS3/sB7WVVgAdcY41nOAuT+bVlR1V2qutR9nQCsAyoAnYDxbrbxQGf3dSdggjoWAEVFpJyvOmwM1xjjaREBGMQVkapAI2AhUEZVd4ETlEWktJutArA93cd2uGm7MivXAq4xxtOyEW9LisjidO9HqurIc8uTWOBT4CFVPexj/DejHeqrARZwjTGeJdlbvGavqjb1XZ7kwwm2H6nqZ27ybhEp5/ZuywHxbvoOoFK6j1cEdvoqP9MxXBGJ87VldWTGGBMMEeLflhVxIvcYYJ2qvp5u13Sgt/u6NzAtXXovd7ZCM+BQ2tBDZnz1cNfgdI/TNzXtvQKVsz4EY4wJrDy8tbcF0BNYJSLL3bTHgaHAFBHpC2wDbnb3fQ10BDYDx4A7sqog04CrqpUy22eMMeFAcGYq5AVVnUfG47IAV2aQX4GB2anDr2lhItJNRB53X1cUkSbZqcQYYwIlr4YUgiHLgCsibwNtcbra4HSdRwSyUcYY4xc/7zILl/UW/JmlcJmqNhaRZQCqul9Ecr9UuzHG5IEwiaV+8SfgnhSRCNz5ZSJSAkgNaKuMMcYPQmBufAgUfwLuOzjz0kqJyL+BW4B/B7RVxhjjp/NqAXJVnSAiS4Cr3KSbVXV1YJtljDFZy8bCNGHB3zvNIoGTOMMKtuCNMSZseGlIwZ9ZCk8Ak4DyOLeuTRSRIYFumDHG+EP83MKBPz3c24EmqnoMQESeB5YALwayYcYY449wmfLlD38C7taz8kUBvwemOcYY4z9nlkKoW+G/TAOuiAzDGbM9BqwRkRnu+3bAvOA0zxhjfBD/FhcPF756uGkzEdYAX6VLXxC45hhjTPacF0MKqjommA0xxpjsOm+GFNKISHXgeaAOEJ2Wrqq1AtguY4zxi5d6uP7MqR0HjMX5x6QDMAWYHMA2GWOM37w0LcyfgFtQVWcAqOpvqvokzuphxhgTUiIQGSF+beHAn2lhSe6jJ34TkXuAP4HSWXzG5MLxpJN0vvctTpxMJjkllevaNmDQXR35afEGnn17OqmqFIrJz5tP9qBaxVKhbm6OFMgfxVcjH6JAvigioyKZPnsZQ0d+fUaeyxpV54WHb6JujfL0fWIs0+csz6Q0/xWNK8j7L9xJ5XLF2bZrP3cMGcOhhERubt+UB3tdDcDRxCT+OfRjVm/6M9f1hVJKSirX9X+dsiWLMPalfsxbspEX3p2OqlIwpgCvDelOVY9+f9I734YU/g+IBR7AeQRFP+DOQDbKFxFJEZHlIrJCRJaKyGXZ/PwzIvJIoNqXFwrkj+LT4fcxZ8JjzB4/iO8XrGfJ6i089sonvPNMT2aPH0SXq5swbNzMUDc1x5JOJNNpwFtc3mMorW57kSub16Fpvapn5Nn+1wEG/vsD/jdjccaF+NCicU3eefr2c9L/r/fVzF20gaY3PsvcRRv4v97tANi6cx/X3v0GLW97kVfGfMuwx7vn6LjCyfv/m0uNKmVOvX/ytf/x5lM9+eb9R+l0VWOGT/guhK3LO2nrKWS1hYMsA66qLlTVBFXdpqo9VfUGVf05GI3LRKKqNlTVBsAQzsM73kSEQgULAHAyOYXk5JRTTyc9cvQ4AAlHj1O2pLef5Xk08QQA+aIiyRcVifPEktO279rPms07SdVznzx9/+1XMnv8o8ybOITB/Tv6XWeH1vWZ9OVCACZ9uZCObeoD8OvKPziUkAjAolV/UL500RwdU7jYFX+QOfPX0u3aZqfSRODIsdPfnzIli4SqeXlGECLEvy0c+LrxYSo+nrGuql0D0qLsiQMOwKlnyU8DigH5gCdVdZq77wmgF7Ad2INza3JYS0lJpd2dr/LHjj3c0fVyGtetymuDu9Hjn/8lukA+YgtF8/Woh0PdzFyJiBB++OAxqlUsxZhP5rJkzVa/Ptf20tpcULk0V/Z+BRFh0mt3c1mj6vyy7LcsP1u6eGF27zsMwO59hylVrPA5eXp2uoxZv6zN3sGEmX8Pn8rjA67nyLGkU2kvDbqVPoNGOt+fgtF8PuKhELYwj4RR79UfvsZw3w5aK7Inxn2iZjRQDrjCTT8OdFHVwyJSElggItOBxkA3oBHO8S4lk4ArIv2B/gCVKoX2ocSRkRHMHj+IQwnHuGPIGNb9tpORH//AR6/dTeO6VXnno9k8/dZUXh/i3Z++qalKqx5DiYuN4cNX+nFR9XKs+83nU6YBaNvsIq64tDZzPxoMQKGYAlxQqTS/LPuN78Y+QoH8URSKKUCxuIKn8jwzfBpzFqzLsuyWTWpy+w3N6dBvWO4OLoRm/7KGEsUKc/GFlZi/bPOp9NFTfmTcy/1pVKcKIybN4bm3P+flx7qFsKV5w0tjuL5ufJgdzIZkQ6KqNgQQkebABBGphzPz4wURaYXzRIoKQBngcmBqusV3pmdWsKqOBEYCNGrcNNPefTAVKVyQyxrVYM6CdazZ9CeN61YFoNOVjej+8PnxaLnDRxKZt2QTVzav41fAFYFh42Yybuq5I1tX3/Eq4Izh3nb9pQz894dn7I/fn0CZEnHs3neYMiXi2HMg4dS+ujXK89aTt3Hzg+9x4NDRXB5V6Cxe9Qezfl7NDwvWknQimYSjx+kzaCS/bYunUZ0qAFx/RSN6PfLfELc09wSI9FDA9fTatqo6HygJlAJ6uH82cQPybk7fqBEWwdNfew8c4VDCMQASk07w0+KN1KxahoSjx/ltWzwAcxdtoFbVMr6KCWslisYSFxsDQHSBfLS55EI2bdnt12fnzF9HjxuaUyjGebReuVJFKFks1q/Pfjt3Fd2vuxSA7tddyjc/rgSgYpliTHi5H/c8PeHUOfaqx+6+joWfPsPPU/7F8Kd7cVnjmox+oS8JR4/z+3bn2H5atOGMC2pe5qWn9vq7AHlYEpHaOIuj7wOKAPGqelJE2gJV3GxzgXEiMhTneK8Hwvqf9vh9h3jguY9ISU0lNVW54cpGtGtRj1cH30rfx98nIkIoUrggb3j4SnrZknG8+0xPIiMiiIgQps5ayox5qxly97UsX7eNb+auolGdynzwcj+KxhWkfcuLGXz3tVx26/N8v3A9taqVZeb7zmSTI8eSuPtf49l74EiW9Q4b/x1jX7yT229ozo7dB+gz2LmD/dG7OlC8SCFefexWAJKTU7mi98uBOwFBFhUVydBHb+GeJ8e5358YXhns/eEECJ9g6g85+8pwphlFCqhqUtY5A0tEUoBVaW+Bx1X1K3fc9gucC2bLcaawdVDVLekumm0FdgBrVfVVX/U0atxUf/h5YaAOw/PKXvZgqJsQ9rbO9e44cDC0a92M5UuX5Cpclq1ZT3u8/qlfeV+/ofYSVW2am/pyy5+1FC4BxuD0ICuLSAPgLlW9P9CNy4iqRmaSvhdonsm+53HWgzDGnGe81MP1Zwz3LeA6nJ/tqOoK7NZeY0yY8NKND/6M4Uao6tazpl6kBKg9xhjjNwGiwiWa+sGfgLvdHVZQEYkE7gc2BrZZxhjjHw/FW78C7gCcYYXKOFOtZrlpxhgTUhJGt+36I8uAq6rxOHdqGWNM2PFQvPVrlsIoMrhxQFX7B6RFxhiTDXk1S0FE3seZIBCvqvXctGdwVkjc42Z7XFW/dvcNAfriXNN6IG3dcF/8GVKYle51NNAFZxEYY4wJKYG8XFx8HM4aMhPOSh929rx9EamD88u/LlAemCUitVTV54QCf4YUPj6rog+A82MhTWOMt+XhbbuqOldEqvqZvRMw2b0Z7A8R2QxcAsz39aGcrKVQjdO3zRpjTEiJn/8BJUVkcbrN32HR+0RkpYi8LyLF3LQKnPlLf4eb5pM/Y7gHOD2GGwHsBwb72VBjjAmYbD4mfW8Obu19D3gOJwY+B7yG88SbjGrNcp0EnwHXfZZZA5znmAGkqr+LLxhjTBAE8tZeVT21hJ07geBL9+0OoFK6rBWBnVmV53NIwQ2uU1U1xd0s2BpjwoqI+LXlsOxy6d52AVa7r6cD3USkgIhUA2oCv2ZVnj+zFH4VkcaqujTbrTXGmAByHpOeV2XJJKANzljvDuBpoI2INMQZLtgC3A2gqmtEZAqwFkgGBmY1QwF8P9MsSlWTgZZAPxH5DTiKM3ahqto4F8dmjDF5Iq/uNFPVjBaYHuMjf7ZXIfTVw/0V53lgnbNToDHGBEs2L5qFnK+AKwCqmvWjUI0xJkTOl1t7S4lIps/hVtXXA9AeY4zJBiEiwxla4clXwI0EYsl4vpkxxoSccP70cHep6rNBa4kxxmSXQJSHBnGzHMM1xphwdT71cK8MWiuMMSaHzosFyFV1fzAbYowxOeGheOvXnWbGGBOWhJwteRgqFnCNMd4l58mQgjHGhDvnTjMLuMYYExTeCbcWcI0xHuehDq4FXGOMl+V8rdtQsIBrjPEsm6VgjDFBZBfNzgMRAgXyRYa6GWFr4fQXQ92EsFel43OhbkJYS9q0K/eFCDakYIwxwWBDCsYYE0TWwzXGmCDxTri1gGuM8TABIq2Ha4wxweGheGsB1xjjZYJ4aFDBAq4xxtOsh2uMMUHgTAvzTsS1gGuM8S6xHq4xxgSN3dprjDFB4CxAHupW+M8CrjHG02yWgjHGBImHRhQ8te6DMcacQ/z8L8tyRN4XkXgRWZ0urbiIfCcim9w/i7npIiJvichmEVkpIo39aasFXGOMZ6WN4fqz+WEc0P6stMHAbFWtCcx23wN0AGq6W3/gPX8qsIBrjPEuESL83LKiqnOB/WcldwLGu6/HA53TpU9QxwKgqIiUy6oOC7jGGE8TPzegpIgsTrf196P4Mqq6C8D9s7SbXgHYni7fDjfNJ7toZozxLGdIwe+rZntVtWkeVn02zepD1sM1xnhaNnq4ObE7bajA/TPeTd8BVEqXryKwM6vCLOAaY7wtsBF3OtDbfd0bmJYuvZc7W6EZcCht6MEXG1IwxnhaXt3aKyKTgDY4Y707gKeBocAUEekLbANudrN/DXQENgPHgDv8qcMCrjHG0/LqvgdV7Z7JriszyKvAwOzWYQHXGONtHrrTzAKuMcaznOFZ70RcC7jGGO+y9XCNMSZ4PBRvLeAaY7xMEA91cS3gGmM8zUPx1gKuMca7cnkXWdBZwDXGeJuHIq4FXGOMp9m0MJNjO/46wIBnJhC/7zARIvTu0oJ7urdl1cYd/HPoZI4cS6JyuRKMfK43cbExoW5uyEyaNo9pMxchIlSvUoanHryJ54d/yrrNfxIVGUmdmhUZMrALUVGRoW5qjlQoFcd7Q26kdPFYUlUZ/+Vi/vvpghoLkgwAABAtSURBVDPyFImN5u1BnalWvjjHTyRz/8ufs25LfCYl+id/vkjeG9KVhrXKs/9wInf+ewrbdx+kTZPqPN3/avJHRXIiOYV/jZjBT8v+yFVdecVLY7hhu3iNiJQRkYki8ruILBGR+SLSJQ/K/UFE8mqJtjwXFRXBfx7qysJPnmLm2EcY/b+5rP99Fw/+ZyJPD+zEL5Of4Lq2DRj+wexQNzVk4vcd4uMvfmHc6/cx6e2HSE1VvvtpJde0bsiUdx9m4vAHSTpxkmkzF4W6qTmWnJLKk+99S7M+w2l370ju6nQJF1YpdUaef/ZoxarNf9HyrncZ8OJnvHh/R7/Lr1SmKF8MO/f2/54dG3Mo4ThNbn+T9z75hWfuvhqAfYeO0v3xj2jR9x3uffEzRgy5MXcHmFfcebj+bOEgLAOuOPM8PgfmquoFqtoE6IazBNp5rWzJIjSo7az6VrhQNLWqlmXXnoNs3hbPZY1rANDmktp88f3yUDYz5FJSU0k6cZLklBSOJ52gZPHCtGhaGxFnmlDdWpWI33co1M3Msd37j7Byk7P41JHEE2zctodyJePOyHNh1dLMXfo7AJu276VymaKUKlYIgFuuqs+sd/szd9QAhj18PRF+PmOmQ4uLmDTD+W5N+3EtrRtfAMCqzX/x174EANZtiSc6fxT584XHr4e8eqZZMIRlwAWuAE6o6oi0BFXdqqrDRSRaRMaKyCoRWSYibQF8pMeIyGT3QW8fA575Hb5t5z5WbthBk7pVqX1BOb6ZuwqAabOX8ufuAyFuXeiULlGEHp0vp1Pfl7i294vEFoqmWaNap/YnJ6fwzffLaNa4lo9SvKNSmaLUr1GOJet2nJG++re/uK5VHQAa165ApbJFKF8qjlqVS9Kl7cW0v380rfq9R0qqcvNV9f2qq3zJwvwZ7/xDlZKayuEjSRSPK3hGnhta1WHl5l2cOJmSB0eXO4K3erjhOoZbF1iayb6BAKp6sYjUBmaKSC0f6QOAY6paX0Tq+yg3rBw5lkSvx0bz4sM3Ehcbw9v/6sHgV//Hy6O/oUOri8kXJr2LUDh8JJG5C9cyddSjFC4Uw5CXJvLN98vo0LYRAC+PmEbDulVpVLdaiFuae4Wi8zPh2W4MeecbEo4lnbHvjYk/8eJ9HZg7agBrf9/Nyk1/kZKSSuvGF9CgVjnmjLgbgOj8+dhz4CgAHzzbjSrlipEvKpKKZYowd9QAAEZ8uoCJ3y7LMDJpugcZ1K5aimf6t6ProPHn5AuVMImlfgnXgHsGEXkHaAmcwFlpfTiAqq4Xka1ALXd/RumtgLfc9JUistJHPf1xnsBJpcqVA3Y8WTmZnELvx0Zxc/umXH9FQwBqVS3LZ2/fB8DmrbuZOW9NyNoXaouWb6Z8meIUKxILQNvmdVm1fisd2jZi9KRZHDh0lJeG9AhxK3MvKjKC8c9245NZK/nyp3Xn7E84lsR9L39+6v2KSf/H1l0HaV6/KpNnLOfZ0bPO+UzPf00GnF7zu4O7cP3/jT1j/849h6lQugg79x4mMiKCuNgCHDicCED5knF88Gx3Bgz9jC07w+gXlocibrgOKawBTj3nXVUH4qxJWYrMT6+v057ls4bcekaqalNVbVqqZKmsPxAAqsr9z31EraplGdjj9DKce/Y742epqam8+v4M7rixZUjaFw7KlCrC6g3bOJ50AlVl0YrNVK1UmmkzF7Fg2Saee6QbERHh+tX23/BBndm4dQ/vfvJLhvvjCkWTz52F0evaJvyycisJx5KYu/R3bmhdl5JFnfHcooVjqFSmiF91fvvLerpf4/wj36l1Hea6MxHiCkXz8dDbeXb0LBau3pbbQ8tTefXU3mAI1x7uHOAFERmgqmnPe08bSJoL9ADmuEMGlYENfqR/LyL1AP8Gs0JkwYrf+fjrX6lTozyX3/YiAE8NvIHft8Uz+n9zAbiuTUN6XN8slM0MqXoXVuaKFvXo9dDbREZGUOuCcnS+5hLa3Pw0ZUsX5a5BzlemTfO63NXtnLWjPaFZvcp0a9eQNb/9depn/3OjZ1GxtBM4x36xmAurlOK9IV1JSU1lw5Y93P+K09vdsHUPz78/m89e6UWECCdTUnn0jS/Zvjvri4gffLWUEY93ZcmHD3LgcCJ9n/sEgH5dLqVa+eI82rM1j/ZsDUDXRyew9+DRQBx+toRHKPWPOAuXhx/3gW3DgEuBPcBRYATOM4VGAE2AZOBhVf1eRKIzSY8BxgJ1gOVADeABVV3sq/4mTZrqzwt9Zvlb27grIdRNCHuX3vZKqJsQ1pKWjyY1YWeu4mW9Bo31s5nz/Mp7YdlCS/Lwqb05Eq493LRnwHfLZHefDPIfzyQ90Uc5xhgPswXIjTEmWMJoypc/LOAaYzzNQ/HWAq4xxstsAXJjjAkaD8VbC7jGGO+yBciNMSaYPBRxLeAaYzzNpoUZY0yQ2BiuMcYEg4CfS/2GBQu4xhiP807EtYBrjPGstAXIvcICrjHG0/Iy3orIFiABSAGSVbWpiBQHPgaqAluAW1Q1RwsCe3/RUGPM31oAHrHTVlUbpltZbDAwW1VrArPd9zliAdcY42lpDw7NasuFTkDaM4XGA51zWpAFXGOMp4mfG1BSRBan2/pnUJziPA9xSbr9ZdzlYtOWjS2d07baGK4xxrOyOVyw148FyFuo6k4RKQ18JyLrc9XAs1gP1xjjaeLnf/5Q1Z3un/HAVOASYLf7BJq0J9HE57StFnCNMd6WjTEFn8WIFBKRwmmvgXbAamA60NvN1hvnMV85YkMKxhhPy8NpYWWAqe4Ftihgoqp+KyKLgCki0hfYBtyc0wos4BpjPCzvHoGuqr8DDTJI3wfkyeOfLeAaYzzLa3ea2RiuMcYEifVwjTGe5qUergVcY4yn2QLkxhgTDNlfJyGkLOAaYzzLaxfNLOAaYzzNhhSMMSZIrIdrjDFB4qF4awHXGONxHoq4FnCNMZ4lkGe39gaDqGqo2xCWRGQPsDXU7UinJLA31I0Ic3aOfAu381NFVUvlpgAR+RbnuPyxV1Xb56a+3LKA6xEistiPxZP/1uwc+WbnJ/RsLQVjjAkSC7jGGBMkFnC9Y2SoG+ABdo58s/MTYjaGa4wxQWI9XGOMCRILuMYYEyQWcANIRLqIiIpI7VC3xetEJEVElovIChFZKiKXZfPzz4jII4FqXyiISBkRmSgiv4vIEhGZLyJd8qDcH0TEpo8FgAXcwOoOzAO65bYgEYnMfXM8LVFVG6pqA2AI8GKoGxRK4jxa9nNgrqpeoKpNcL5nFUPbMuOLBdwAEZFYoAXQFzfgisjHItIxXZ5xInKjiESKyCsiskhEVorI3e7+NiLyvYhMBFa5aZ+7vZk1ItI/XVl9RWSj2zsZJSJvu+mlRORTt+xFItIieGchYOKAA+CcZxGZ7fZ6V4lIp7RMIvKEiGwQkVnAhaFqbIBcAZxQ1RFpCaq6VVWHi0i0iIx1z8cyEWkL4CM9RkQmu9+9j4GY0BzS+c/WUgiczsC3qrpRRPaLSGNgMnAr8LWI5Md59PIAnKB8SFX/ISIFgJ9FZKZbziVAPVX9w31/p6ruF5EYYJGIfAoUAJ4CGgMJwBxghZv/TWCYqs4TkcrADOCiAB97IMSIyHIgGiiHE3AAjgNdVPWwiJQEFojIdJxz0Q1ohPM9XwosCX6zA6YuzjFlZCCAql7sDmfNFJFaPtIHAMdUtb6I1PdRrsklC7iB0x14w3092X3/FPCWG1Tb4/wcTBSRdkB9EbnJzV8EqAmcAH5NF2wBHkg3TlfJzVcW+FFV9wOIyCdALTfPVUAdOb3AR5yIFFbVhLw93IBLVNWGACLSHJggIvVw1i95QURaAalABaAMcDkwVVWPuZ+ZHppmB4eIvAO0xPnO7ACGA6jqehHZivN9aJlJeivgLTd9pYisDP4R/D1YwA0AESmB0wOrJyIKRAIKDAJ+AK7B6elOSvsIcL+qzjirnDbA0bPeXwU0V9VjIvIDTo/P13JJEW7+xNweV7hQ1flub7YU0NH9s4mqnhSRLTjnBJxzfr5aA9yY9kZVB7rnZDHwZyaf8fU9OZ/PVdiwMdzAuAmYoKpVVLWqqlYC/sDpYUwG7sDpgaUF2BnAABHJByAitUSkUAblFgEOuMG2NtDMTf8VaC0ixUQkinT/IwIzgfvS3ohIwzw7yhBxjz0S2IdzTuLdYNsWqOJmmwt0cccnCwPXh6a1ATMHiBaRAenSCrp/zgV6gPNdAioDG/xMrwfUD0L7/5ashxsY3YGhZ6V9CtwGPABMAKar6gl332igKrDUvfq8B2cM+GzfAve4P/k2AAsAVPVPEXkBWAjsBNYCh9zPPAC8434mCud/rnvy4BiDLW0MF5yeWm9VTRGRj4AvRGQxsBxYD6CqS90LQMtxltn8KRSNDhRVVRHpDAwTkUE435mjwGPANGCEiKwCkoE+qpokIu9mkv4eMNb9jizH+QfcBIDd2nueEJFYVT3i9nCnAu+r6tRQt8sYc5oNKZw/nnF7gKtxhi8+D3F7jDFnsR6uMcYEifVwjTEmSCzgGmNMkFjANcaYILGAa3JETq/etVpEPhGRgll/KtOy2ojIl+7rG0RksI+8RUXk3hzUkeFqYZmln5VnXLq7AP2pq6qIrM5uG835zwKuyam01bvq4dxOesbcXnFk+/ulqtNV9ew5zOkVBbIdcI0JBxZwTV74Cajh9uzWuRPslwKVRKSdOOu0LnV7wrEAItJeRNaLyDyga1pBItJHTq90VkZEpoqzBu4KcdbAHQpUd3vXr7j5HpXTK639O11Z2VotTET6ueWsEGeFtfS99qtE5CdxVmS7zs2f4SpvxmTGAq7JFfdGiw64y0fiBLYJqtoI586nJ4GrVLUxzn3+D4tINDAK53bby3EW38nIWziL8jTAWf1rDTAY+M3tXT/qLvxTE2dVtYZAExFpJSJp68M2wgno//DjcD5T1X+49a3DWcUtTVWgNXAtzt1a0aRb5c0tv5+IVPOjHvM3Zbf2mpxKf6vtT8AYoDywVVUXuOnNgDo4y00C5AfmA7WBP1R1E4CIfAj051xXAL0AVDUFOCQixc7K087dlrnvY3ECcGGyv1pYPRH5D86wRSyn17oAmKKqqcAmEfndPYbMVnnb6Edd5m/IAq7JqVPLJaZxg+rR9EnAd6ra/ax8Dcm71akEeFFV/3tWHQ/loI5xQGdVXSEifYA26fadXZaS+SpvVbNZr/mbsCEFE0gLgBYiUgNARAq6q1StB6qJSHU3X/dMPj8bZ3HstPHSOJwF1gunyzMDuDPd2HAFESlNzlYLKwzsEmfVth5n7btZRCLcNl+As3iQv6u8GQNYD9cEkKrucXuKk8RZdB3gSfcpGP2Br0RkL85z3+plUMSDwEgR6QukAAPctXB/dqddfeOO414EzHd72EeA23O4WthTOCuubcUZk04f2DcAP+Isbn6Pqh4XEX9XeTMGsLUUjDEmaGxIwRhjgsQCrjHGBIkFXGOMCRILuMYYEyQWcI0xJkgs4BpjTJBYwDXGmCD5fwf6sB1y/QO0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Best parameters set found on development set:\\n\", clf.best_params_)\n",
    "print(\"\\n Best Score\", clf.best_score_) \n",
    "\n",
    "print(\"\\n Grid scores on development set:\")\n",
    "means = clf.cv_results_['mean_test_score']\n",
    "stds = clf.cv_results_['std_test_score']\n",
    "for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "    print(\"%0.3f (+/-%0.03f) for %r\" % (mean, std, params))\n",
    "\n",
    "print(\"Detailed classification report:\")\n",
    "print(\"The scores are computed on the full evaluation set. \\n\")\n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(classification_report(y_true, y_pred))\n",
    "\n",
    "#Lets print confision matrix\n",
    "labels =['Average','Bad','Good']\n",
    "disp = plot_confusion_matrix(clf, X_test, y_test,\n",
    "                                 display_labels=labels,\n",
    "                                 cmap=plt.cm.Blues)\n",
    "\n",
    "print ('\\nTesting score: ',clf.score (X_test, y_test)) # print the testing score(accuracy)\n",
    "print ('\\nTraining Score: ',clf.score (X_train, y_train)) # print the trainig score(accuracy)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cross Validate\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.metrics import recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#start 18:37"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:    7.1s remaining:   10.7s\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   5 | elapsed:    7.2s remaining:    4.8s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 76.1 ms, sys: 69.8 ms, total: 146 ms\n",
      "Wall time: 7.48 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:    7.5s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:    7.5s finished\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "scoring = ['f1_macro']\n",
    "scores = cross_validate(clf, X, Y, cv=5, scoring=scoring, verbose=10, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "['fit_time', 'score_time', 'test_f1_macro']\n",
      "test_f1_macro [0.57448591 0.56692647 0.6030178  0.56020195 0.59297879]\n"
     ]
    }
   ],
   "source": [
    "print(\"-------------------------------------\")\n",
    "print(sorted(scores.keys()))\n",
    "print('test_f1_macro', scores['test_f1_macro'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K best Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 0 0 0 0]\n",
      " [0 1 0 0 0]\n",
      " [0 0 0 0 1]\n",
      " ...\n",
      " [1 0 0 0 0]\n",
      " [0 0 0 1 0]\n",
      " [1 0 0 0 0]] ['Bad' 'Bad' 'Bad' ... 'Average' 'Bad' 'Average']\n",
      "4800 4800 1200 1200\n"
     ]
    }
   ],
   "source": [
    "#Load Kbest data\n",
    "arr2= np.load('data_kBest.npy', allow_pickle= True)\n",
    "X= arr2[:,0:-1]\n",
    "Y= arr2[:,-1]\n",
    "\n",
    "\n",
    "### Split arrays or matrices into random train and test subsets - 20% for Test, 80% to train\n",
    "X_train, X_test, y_train, y_test= train_test_split(X, Y, test_size=0.20,  random_state = 200)\n",
    "\n",
    "print(X_train,y_train)\n",
    "print(len(X_train),len(y_train),len(X_test),len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.66 s, sys: 0 ns, total: 4.66 s\n",
      "Wall time: 4.65 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score=nan,\n",
       "             estimator=LinearSVC(C=1.0, class_weight=None, dual=True,\n",
       "                                 fit_intercept=True, intercept_scaling=1,\n",
       "                                 loss='squared_hinge', max_iter=1000,\n",
       "                                 multi_class='ovr', penalty='l2',\n",
       "                                 random_state=None, tol=0.0001, verbose=0),\n",
       "             iid='deprecated', n_jobs=None,\n",
       "             param_grid=[{'C': [0.1, 1, 10, 100, 1000]}],\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1_macro', verbose=0)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "tuned_parameters = [{'C': [0.1,1, 10, 100, 1000]}]\n",
    "clf = GridSearchCV(LinearSVC(), tuned_parameters, scoring='f1_macro')\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters set found on development set:\n",
      " {'C': 0.1}\n",
      "\n",
      " Best Score 0.4688739965134472\n",
      "\n",
      " Grid scores on development set:\n",
      "0.469 (+/-0.020) for {'C': 0.1}\n",
      "0.469 (+/-0.020) for {'C': 1}\n",
      "0.469 (+/-0.020) for {'C': 10}\n",
      "0.458 (+/-0.046) for {'C': 100}\n",
      "0.361 (+/-0.044) for {'C': 1000}\n",
      "Detailed classification report:\n",
      "The scores are computed on the full evaluation set. \n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Average       0.46      0.28      0.35       405\n",
      "         Bad       0.53      0.49      0.51       392\n",
      "        Good       0.49      0.72      0.58       403\n",
      "\n",
      "    accuracy                           0.50      1200\n",
      "   macro avg       0.50      0.50      0.48      1200\n",
      "weighted avg       0.50      0.50      0.48      1200\n",
      "\n",
      "\n",
      "Testing score:  0.48276523868797655\n",
      "\n",
      "Training Score:  0.46904203445488185\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEGCAYAAAApAy29AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3wUVdfA8d9JQkJJIZCE3ouISLdgRUTF3hX0wYaiSHntYnvE3h5BBQSxIYqij4BieRSVroLSERGkSk8gERICpJ33j5mEACkbki2D5+tnP8neuTtz77ic3D17546oKsYYY/wvLNgNMMaYfwoLuMYYEyAWcI0xJkAs4BpjTIBYwDXGmACJCHYDQlX1GjW1dr2GwW5GyJJgN8ADdmRmBbsJIS09ZQv7dqeV660UHttINWevT3V1b8q3qtqjPMcrLwu4xahdryHvTJoW7GaErDCxkFuadxZtCnYTQtrkwdeWex+as5eoY67xqe6+xSMTyn3AcrKAa4zxMAHxTmbUAq4xxrsECAsPdit8ZgHXGONtHkpvWcA1xniYpRSMMSZwbIRrjDEBINgI1xhjAkNshGuMMQFjsxSMMSYQ7EszY4wJDMFSCsYYEzA2wjXGmECwlIIxxgSGAOH2pZkxxgSG5XCNMSYQLKVgjDGBYyNcY4wJEBvhGmNMAIhd2muMMYFjl/YaY0wg2JdmxhgTOJZSMMaYALD1cI0xJlAspWCMMYFjX5oZY0yAWA7XGGMCQCylYIwxgWMjXGOMCQyxgGuMMf7n3GHHAq4xxvifCBJmAfcf76XXJzN34Uqqx1Xj7ZcHHrb9+9lLmPD5bACqVI7krlsvplnjOuU6ZlZ2Di+MmMiqtVuIjanKY3ddQ+2keOYvXc1b478jJyeHiIgIbu99Hh3aNC3XscrrhdcnMXeBc37eHTrosO1/bU7hhZGT+HPdFvr0OodrLzmt3MfMys7hueGfFpyfx+++1jk/S1YzZvxUcnJyiYgI547e59Hx+GblPl55XdexHsfVjiF9fw7P/7D6sO2VI8K4oXN94qtWIkyEaX/uYN5ff5frmFUrhXPTiQ2oUbUSqZnZvPvLX+zNzqNz/TjObpkIQFZOHh8v3sKW3fvKdayK4qURrl+/3hORy0VERaSVP48Tis7r2oHnHr6h2O11kuIZNqQPb/1nAP+6sitDx0zxed/bktO4Z8jbh5X/b9oCoqtV4f3hd3PlhV14c/xUAOJiqvL0g9fz1ssDebD/FTw3/NOyd6iC9ejagRceubHY7THRVRh4y4Vcc3HZA+225DTuevytw8q/nraAmOgqjB9xD1dfdApvfPAtAHGxVXl28L94Z+hAHhpwZUicH4B5G9IY9eP6Yref3rQm29L388K0NQyfvY7Ljq9NuI/Bp3lCNa7vWO+w8u4tE1iVksHT3/3JqpQMznGD7M7MLF6bvZYXpq3mm5XJ9OxQ94j65A8i4tMjFPh7PkUvYA7Qs7w7EhHvzG4G2rZuTGx0lWK3H3dMQ2Lc7a1bNCBl566Cbd/NWsydD42m7/0jGTrmc3Lz8nw65k/z/+Dcru0BOPPk41j421pUlRZN6pJQIxaAxg2SyMrOISs750i7ViHatW5S4vmJj4umVfP6REQc/hb9btZi+g0exa33jeDlNz4jN9e38/Pjrys478wOQOifH4A1OzPJzM4toYYS5Z6fyIgwMrNyyVMFoFuLBO7t2pQHuzXn/FZJPh/z+Dqx/LLBGSX/suFvjq/jnJd1qXvZm+2c5/WpmVSvUukIeuQfFRVwRaSBiEwXkRUislxE/s8tHyIim0Vksfu4oNBrHhKR1SKyUkTOK+0Yfgu4IhINnAr0wQ24IvLxIY0dKyJXiki4iLwkIr+KyFIRud3d3tU9AR8Cy9yyz0RkgXtC+hbaVx8RWSUiM0TkTREZ4ZYnishEd9+/isip/urzkfrftAWc2KElABs2JTPjp9947anbGPNSf8LDwvhh9hKf9rMjdTdJNeMACA8Pp1rVKHanZx5UZ9a85bRoUofISt7MJm3YlMz0n5Yx/Om+vPWfAYSFhfH9nDKcn4QD5ye6qPMzdznNPXJ+Zq1NpXZMFE+dfwwPnd2ciUu3okCrpGgSq0Xy8oy1vDhtNQ3iK9OsZlWf9hkTFcHu/c4fm937c4iJOvw8dGkUz4rt6RXZlSMnZXiULge4V1WPBU4G+otIa3fbMFVt7z6+BnC39QSOA3oAr5c2MPTnu+oy4BtVXSUiqSLSEZgAXAt8LSKRwNlAP5ygvEtVTxCRKOBHEZnq7udEoI2qrnOf36KqqSJSBfhVRCYCUcBjQEcgHZgG5P8rfBXnZM0RkYbAt8Cxfux3mSz6bS3/m76AV568reD5n+u2cOdDowHYn5VN9dhqAPz7pQ/ZlpxGdk4uyTt20ff+kQBccUEXepzVEXVHNwcp9Jd9/cbtvDl+Ki8+cpN/O+VHC5etZdXaLdwxeBQAWVk5xMc55+exF8ezNTmNnJxctu/Yxa33jQDgygu7cP5ZnSjq9BQ+P+s2bmfM+G958dGb/N2NCnFsUjSbdu1j+Jz1JFSLpP+pjXlh2mqOSYqmVVI0D5zl5KGjIsJIjI5izc5M7jmzKRFhQlREGFUjwwvqTFm+nT+SM0o9ZouEapzcOJ5XZq0rtW4gCBWXLlDVrcBW9/d0EVkBHJ53OeBSYIKq7gfWichqnHj1c3Ev8GfA7QW84v4+wX3+GPCaG1R7ALNUda+InAu0FZGr3PpxQAsgC/ilULAFGCQil7u/N3Dr1QZmqmoqgIj8F2jp1ukOtC70PyVWRGJU9bA/0e6IuS9Arbr1y9V5X6zZsI2X3/iM5x66gbgYZwSiCuee2Z5brzv3sPpP3n8d4OQoX3x9EkOH9Dloe2LNOJJ37iKxZhy5ubnsydxf8LE9Zecu/v2fjxjc/0rq1q7h5575j6Kcd2YHbrv+8PPz1APXA875eX7kRF554taDtifWjCV5x4Hzk3Ho+XnpQwYPuIp6tWv6vyMV4KRG8Xy3KgWAHXuy2JmZRVJMFAJ8tyqFn9anHfaaoTPXAk4O96SG1Rm/cPNB29P35xDrjnJjoyJI338gtVI3NopeHeox6uf1ZGaVlOoIrLCwiv+gLiKNgQ7APJxP6gNE5AZgPs4oOA0nGM8t9LJNlByg/ZNSEJGaQDfgLRFZD9yPM7LdD8wAznOfT8h/CTCw0JC9iarmj3D3FNpvV5wA2kVV2wGLgMqU/IEhzK2fv+96RQVbAFUdo6qdVbVz9RoJR9J1n23f8TdD/vMRDw24igZ1Dxyrw/FNmTV3OWm7nNHG7oxMtqf49s1zl06tmDpjMQAz5y6nw3FNEBEy9uzl4eff59Ze59CmVaOK70wAdWzTjJmFz096JttSDg8sRTmlcyu+nbkIcM9Pm6YF52fwc+9z63XncryHzk9aZhbHJEYDEBMVTlJ0FDv3ZPFHcgYnN4onMtz55x1XOYLoSN++Avlt225ObFQdgBMbVWfZ1t0AxFepRJ+TGvL+go2kZGT5oTdHrgw53AQRmV/o0beY/UUDE4G7VHU3MApoBrTHGQG/nF+1iJcX9TmqgL9GuFcB41T19vwCEZkJnIYTZG8FOgM3uZu/BfqJyDRVzRaRlsDBf3odcUCaqma6Mx9Odst/AYaJSDxOSuFK3JwvMBUYALzktqO9qi6usJ4W4+lXPmHJ7+vYlZ7JtXe8xI3XdCM3xxkVXHzuibz/6Qx2Z2Ty6ltfABAeHsao5/vRuH4SN/fszoNPv0eeKhHh4QzqcxG1EquXeswLunXkuRET6T1wGDHRVXj0rmsA+OybeWzZlsoHE2fwwcQZALzw6I3Ex0X7p/M+eOqVj1m83Dk/V9/+Ijdd063gy69Lzj2R1LR0bh88isy9+xERPv3qJ8YOG0TjBknc0rM79z81FlUlPDycu269mNqJ8aUe88JunXh2+KdcP2AosdFVeOzuawGY/M1ctmzbyfufTuf9T6cD8NJjNwX1/ADc2Lk+zROrER0ZwZM9juHrFcmEu//Ef1yfxjcrU/hXx/oM7tYcBKYs38aerFz+SM6gVkwU95zpTP3bn5vH+/M3keHDqPS7VTu4+YQGnNwonrTMbN79ZSMAPVolUi0ygqvbObMT8hT+M2ONfzpeFr7nZwF2qGrnEncnUgkn2I5X1UkAqrq90PY3gS/dp5twPmXnqw9sKXH/Reb9yklEZgDPq+o3hcoG4eROBwHbgCmqerO7LQx4GrgY5/Sl4OSAOwD3qepFbr0o4DOcYftKIBEYoqoz3L9W9+F0eAWQqqqPiEgCMNI9dgROGuOO0vrQ6vgO+s6kaeU9FUetsBCZZhPK3lm0KdhNCGmTB19Lyprl5XojRSQ01eoXPetT3Z3v9VpQUsAVZxj8Hk7suKtQeR03v4uI3A2cpKo9ReQ44EOcvG1d4AeghaoW+5fNLyNcVe1aRNlrhZ7WPGRbHvCw+yhshvvIr7cfOL+Yw36oqmNEJAKYjDOyRVV34KQvjDFHmYr80gwnV9sbWCYi+Z+CHwZ6iUh7nHTBeuB2AFVdLiKfAL/jzHDoX1KwhaPrSrMhItIdJ6c7FWckbIw5ylXUpb2qOoeiExRfl/CaZ4BnfD3GURNwVfW+YLfBGBNg4q1Le4+agGuM+WeygGuMMQFiAdcYYwKggr808zsLuMYYb/NOvLWAa4zxMPHPpb3+YgHXGONpllIwxphA8U68tYBrjPE2G+EaY0wA+Ho3h1BhAdcY42kWcI0xJkDsNunGGBMgNsI1xphAsMVrjDEmMISD7gMa8izgGmM8zGYpGGNMwITZl2bGGBMAYikFY4wJCMFGuMYYEzA2wjXGmACxL82MMSYQLIdrjDGBIYgtQG6MMYFiI1xjjAkQy+EaY0wgWA7XGGMCw1lLwTsR1wKuMcbTPBRvLeAaY7zNrjQzxphAsPVwjw5REWE0TqwW7GaErDs/XRrsJoS8Aac2DnYTQtqMKpXKvQ9bD9cYYwLG1sM1xpiA8VC8tYBrjPEwsS/NjDEmIGwerjHGBJAFXGOMCRAPxVsLuMYYb/PSCNc7C0kaY8yh3MVrfHmUuiuRBiIyXURWiMhyEfk/t7yGiHwnIn+6P+PdchGR10RktYgsFZGOpR3DAq4xxrOcBch9e/ggB7hXVY8FTgb6i0hrYDDwg6q2AH5wnwOcD7RwH32BUaUdwAKuMcbTwkR8epRGVbeq6kL393RgBVAPuBR4z632HnCZ+/ulwDh1zAWqi0idEtt6ZF00xpjQUIaUQoKIzC/06Fv8PqUx0AGYB9RS1a3gBGUgya1WD9hY6GWb3LJi2ZdmxhjPkrItXrNDVTuXvk+JBiYCd6nq7hL2X9QGLWnfxQZcEYkt6YWquruk7cYYEwgVeaGZiFTCCbbjVXWSW7xdROqo6lY3ZZDslm8CGhR6eX1gS0n7L2mEuxwnWhfuTv5zBRr63AtjjPGTirq0V5yh7NvAClUdWmjTFOBG4Hn35+eFygeIyATgJGBXfuqhOMUGXFVtUNw2Y4wJBYIzU6GCnAr0BpaJyGK37GGcQPuJiPQB/gKudrd9DVwArAYygZtLO4BPOVwR6Qk0VdVnRaQ+ThJ5QVl6Yowx/lBRKQVVnUPReVmAs4uor0D/shyj1FkKIjICOAsn8oMTyUeX5SDGGOMX4qyH68sjFPgywj1FVTuKyCIAVU0VkUg/t8sYY3wSIrHUJ74E3GwRCcOd7iAiNYE8v7bKGGN8IODTRQ2hwpeAOxJnmkSiiDwBXAM84ddWGWOMj46qBchVdZyILAC6u0VXq+pv/m2WMcaUzteFaUKFr1eahQPZOGkFuxzYGBMyvJRS8GWWwiPAR0BdnCspPhSRh/zdMGOM8YX4+AgFvoxw/wV0UtVMABF5BlgAPOfPhhljjC9CZcqXL3wJuBsOqRcBrPVPc4wxxnfOLIVgt8J3JS1eMwwnZ5sJLBeRb93n5wJzAtM8Y4wpgfi8uHhIKGmEmz8TYTnwVaHyuf5rjjHGlM1RkVJQ1bcD2RBjjCmroyalkE9EmgHPAK2ByvnlqtrSj+0yxhifeGmE68uc2rHAuzh/TM4HPgEm+LFNxhjjMy9NC/Ml4FZV1W8BVHWNqj6Ks3qYMcYElQiEh4lPj1Dgy7Sw/e5K6GtE5A5gMwduomb8YO1fyQx6clzB841bd3LXzT1YuHwD6zY6d/fYnbGX2OgqfPnWfcFqZrncfmpjOtaPY/e+HO7/fPlh26tFhnP7qY2pFRNFdq4y+sd1bPp7X7mOGREm9D+9CU1qViVjfw6vzlxLSkYWx9eJpVenekSECzm5yvj5m1i+Lb1cx6oIr77xGb8uWkVcbDVGvnj4sqsz5ixl4hfOhKHKlSO585aLaNKodrmOmZ2dw9BRk1mzbgsx0VV5YNBV1EqMZ9GyNbz30ffk5OYSER7OzdefQ7vjmpbrWBXFSykFXwLu3UA0MAgnlxsH3OLPRpVERHKBZTifEnKBAar6UxlePwTIUNX/+KeF5de0YVJBIM3NzeOUq5/g3NOO5+arziyo8+zrnxNTrXJxuwh5M1fv4NsVyfQ/vUmR2y9rW4cNqZkMnb6GunGVueWkhjw9dZVP+06MjqTfaU148puVB5Wf1SKBjKwc7pr0G12axHNdp/q8OnMt6fuzeemH1aTtzaZ+9co8fE5L7vzv0nL3sbzOPqM9F557IsNGTS5ye62k6jz32M1ER1dh/uI/GfHWF7z81G0+7Xt7ShqvjP6M5x47+CYFU2csJLpaZcYM+z9m/bSMsR99z4ODriY2piqP3d+LmvGxbNi4nX8//wHvjby33H2sCB6Ktz4tXjPP/TWdA4uQB9NeVW0PICLn4VzxdmbJL/Gunxb+ScO6NalXu0ZBmary1YwlfDC0XxBbVj5/bM8gMbr4ZZXrxVXm82XbANiyax+J0ZHEVY5g174cTmtagx7H1iIiXFidsoe3525AS7xXqqNzw+p8uti5x9+89WncfJJzW771qXsL6mz6ex+VwsOICBNy8nzYqR+1ObYx21PSit1+bMsDtxVs1bw+O1IP3Nd1+pwlfPHNPHJyc2nZrD79brmQ8LDSM4jz5q/kuiu7AnDqSa0ZPfZrVJVmjesU1GlYP4ns7Byys3OoVCm4N/4WxFNrKZR04cNkSrjlr6pe4ZcWlU0skAYFtzb+HIgHKgGPqurn7rZHgBtw7iGfgnNpsid8OW0RF5/d4aCyX5euJSE+mib1E4PUKv/7K20vJzaqzsrkDJolVCMhOooa1SKpFhVBlyY1ePzrP8hV5ZaTG3Ja05rMXrOz1H3WqBrJzj1ZAOQp7M3KJSYqgvT9OQV1TmoUz/rUzKAH27KaOmMhndo1B2Dj5hRm/7ycF4f0ISIinNff+ZKZc5bS7Yz2pe5nZ9puEmo6N+wODw+nWtXK7E7PJC62WkGdn375naaNagc92AJwFK0WNiJgrSibKu4N3ioDdYBubvk+4HL3PvIJwFwRmQJ0BHoCHXD6u5BiAq6I9AX6AtSrH/ybEmdl5/DDT8u5/7YLDyr/YtoiLj67Y5BaFRifL9vKjSc25PlLWrMxbS/rUzPJzVOOrxNDk5pVeebiYwGIDA9j9z4nYN5zVjOSYqKICBMSqkXy/CWtAfjf79uZubrogKyFxhT1q1fmuk71ePa7P/3cu4q1dPk6vpuxiBcedzJ9S35by5p1W7jnsTEAZGXlUN0NmM8MncD2lDRycnJJ2bGLQQ+NAuCS806me9cORX5SKJwj3bApmbEffc+TD4XCh13HUZHDVdUfAtmQMiicUugCjBORNjg53WdF5AycO1LUA2oBpwOTCy2+M6W4HavqGGAMQLsOnYI+xJk57w+Oa1mPhBoxBWU5ubl8O3spn79xTxBb5n97s/MY/eP6gufDrzqelIz9HFsrhlmrdzJh4ebDXjN0+hqg+BxuamYWNatFkpqZTZhAlchwMvbnAlCjaiXuPas5I+esZ3v6fv91rIKt+2sbw9+cwpAHryc2pirgfCztdkZ7buzZ/bD6j9zTEyg+h5tQI5YdO3eTUDOO3Nxc9mTuIya6CgA7du7i2aETuLvf5dSpVeOwfQeDAOEeCrieXttWVX8GEoBE4Hr3Zyc3IG/nwIUaQQ+eR+KLaQu5uNvBI9kfF6yiWYMk6iRWD1KrAqNqZHjBVJ5uLRJYsS2dvdl5/LZ1Nyc1jie2sjNWqBYZTkI1326xt2Dj35zRvCYAJzWOZ/nW9IJjPdi9BR8t3MSq5Aw/9MY/knf8zXPDPuaeOy+nXp2EgvJ2xzXhx3m/8/cupy/pGZkkp/zt0z5P6nQMP8x27hD+47zfaXtcE0SEjD17eeKlD7mhZ3daHxP8T3+FhYlvj1AQAkmYIycirXAWR9+JM3siWVWzReQsoJFbbRYwVkSex+nvxcAbwWhvWezdl8WPC1bxzD1XH1T+5bTFR0U6YeAZTWhdO4aYyhGMvLotny7eUhBgv1+ZQr24ytx5ehPyVNn89z7ecEe7m3ft45OFm3n43JbONJU85Z15f7HDzc2WZPqfO+h/ehNeuaINGftzeW2mMyI+r1UStWKiuKJdXa5oVxeAZ6euKkhVBMtLwz9l2Yr17E7P5KYBL3PdlWeRm+uMyM/vfgITJs1kd/peRr3rLHUSHhbGsGdup2H9JHpf041/P/8+mqeEh4dzx80XkOTDH+lzunZg6OuT6Xv3q0RXq8IDA68C4Kupv7B1eyofT57Jx5NnAvDk4N5Uj4v2U+99FyrB1Beivny9C4hIlKoG/bNWoWlh4HyieFhVv3Lztl/gfGG2GDgVOF9V1xf60mwDsAn4vbRpYe06dNL/Tf/ZX93wvDs/Df60qVA34NTGwW5CSLvz6u6s+m1xucJl7RZt9PqhE32qO/SSVgtUtXN5jldevqylcCLwNs4IsqGItANuVdWB/m5cUVQ1vJjyHUCXYrY9gzOH2BhzlPHSCNeXHO5rwEU4H9tR1SXYpb3GmBCRfyPJ0h6hwJccbpiqbjhk6kWun9pjjDE+EyAiVKKpD3wJuBvdtIKKSDgwEPDtGktjjPEzD8VbnwJuP5y0QkOcqVbfu2XGGBNUIkfJpb35VDUZ50otY4wJOR6Ktz7NUniTIi4cUNW+fmmRMcaUgZdmKfiSUvi+0O+VgctxFoExxpigEgiZxcV94UtK4ePCz0XkfeA7v7XIGGN8FUKX7friSC7tbcKBy2aNMSaoJGTuWFY6X3K4aRzI4YYBqcBgfzbKGGN8cVTdJt29l1k7nPuYAeSpr4svGGNMAHgp4JZ4aa8bXCeraq77sGBrjAkpIuLTIxT4spbCLyLi/fUAjTFHHec26b49St+XvCMiySLyW6GyISKyWUQWu48LCm17SERWi8hK9/6KpSrpnmYRqpoDnAbcJiJrgD04aRNVVQvCxpigq8Arzcbi3Fps3CHlww5dzlVEWuNcEHYcUBf4XkRaqmqJ68yUlMP9Bed+YJeVsdHGGBMQFfmlmarOEpHGPla/FJjgrhG+TkRWAycCJS6iXVLAFbcRa3xsgDHGBFwZBrgJIjK/0PMx7n0MSzNARG4A5gP3qmoazj0T5xaqs8ktK1FJATdRRIq9U6GqDvWhocYY40dCmO/zcHccwR0fRgFP4UyNfQp4GbgFijxoqZMKSgq44UB0MTs2xpigE/y7eI2qbi84lrOuzJfu001Ag0JV6wNbSttfSQF3q6o+eSSNNMaYgBCI8ONEXBGpo6pb3aeXA/kzGKYAH4rIUJwvzVrgfO9VolJzuMYYE6oqcoQrIh8BXXFyvZuAx4GuItIeJ12wHrgdQFWXi8gnwO9ADtC/tBkKUHLAPbtcrTfGmACoqGlhqtqriOK3S6hf5pvTFhtwVTW1LDsyxphgCJGLyHxyJKuFGWNMSBB8u1w2VFjANcZ4l1TolWZ+ZwHXGONZzpVmFnCNMSYgvBNuLeAaYzzOQwNcC7jGGC8LnbVufWEB1xjjWTZLwRhjAsi+NDsKhIcJ8dUqBbsZIWvCTWVddOmfJ/6EAcFuQkjbv7bUtV5KJ1hKwRhjAsFSCsYYE0A2wjXGmADxTri1gGuM8TABwm2Ea4wxgeGheGsB1xjjZYJ4KKlgAdcY42k2wjXGmABwpoV5J+JawDXGeJfYCNcYYwLGLu01xpgAcBYgD3YrfGcB1xjjaTZLwRhjAsRDGQULuMYYb7MRrjHGBIDlcI0xJlBEbJaCMcYEinfCrQVcY4yHOSkF74RcC7jGGE/zTri1gGuM8ToPRVwLuMYYT7OUgjHGBIh3wq0FXGOM13ko4lrANcZ4lmBXmhljTGDYerjGGBM4Hoq3FnCNMV4miIeGuBZwjTGe5qF4S1iwG2CMMUdKyvAodV8i74hIsoj8Vqishoh8JyJ/uj/j3XIRkddEZLWILBWRjr601wKuMcbbKiriwligxyFlg4EfVLUF8IP7HOB8oIX76AuM8uUAFnCNMZ4mPv5XGlWdBaQeUnwp8J77+3vAZYXKx6ljLlBdROqUdgzL4Yaodpc+TnTVKMLDwogID2PauAe45eF3WL0hGYBdGXuJi67CrPGDS9nT0antJf8+cH4iwpg+7kEAxnw8gzc/mUVEeBjnnNaGJwddVsqeQlO9WtUZNeQGkmrGkqfKe5N/5I0JMw6qExdThRGP/Ysm9RPYl5XNwKfGs2LN1nIdN7JSBKOe6E37Vg1J3bWHWx5+h41bU+l6YiseH3AJkZUiyMrO4d+vfcbs+avKdayKUoYcboKIzC/0fIyqjinlNbVUdSuAqm4VkSS3vB6wsVC9TW5Zif8DQjbgikgtYBhwMpAGZAEvqurkcu53BnCfqs4vrW6wTRk1iJrVowuev/PsLQW/P/rKJGKjqwSjWSHji9H/d9D5mT1/FV/PXMacjx4iKrISKanpQWxd+eTk5PHoK5NYunIT0VWjmD7uQWbM+4OV67YV1Ln35vNYtmoTvR94kxaNavHSg9dw2Z3Dfdp/gzo1eP3x3lx8x6sHlfe+tAu7du+l0xVPcMU5nRgy8FL6PPwuO//OoNc9b7Btxy6ObVaHT1/rz3EXPlqhfT4iZZuHu0NVO1fckQ+jpb0oJFMK4szz+AyYpapNVbUT0BOoH9yWhQZV5bPvF3HluZ2C3ZSQ8gbVGsIAAA4RSURBVM7E2dx14zlERVYCILFGTJBbdOS279zN0pWbAMjI3M+q9duok1j9oDrHNKnNrF9XAvDnhu00rFOjoM/XnH8C34+9j1njBzPsoZ6E+XgfmvPPaMtHX80D4PNpizjzhGMAWLZqE9t27AJgxZqtVI6sRGSl0BivVVRKoRjb81MF7s9kt3wT0KBQvfrAltJ2FpIBF+gGZKnq6PwCVd2gqsNFpLKIvCsiy0RkkYicBVBCeRURmeB+k/gx4IlhoQBXDhzJWTe8yNjJPx607edFa0iqEUOzhklFv/gfQES4YsAIuvZ+gbGT5gCwekMyPy9eQ/ebXuLCvq+wcPmGILeyYjSoU4O2x9RnwfL1B5X/9udmLjqrPQAdWzeiQe0a1E2qTsvGtbj8nI706DOUM65/nty8PK7ucYJPx6qbFMfm7WkA5ObmsTtjLzXiqh1U55Ju7Vm6aiNZ2Tnl71w5Cc4I15fHEZoC3Oj+fiPweaHyG9zZCicDu/JTDyUJjT9RhzsOWFjMtv4Aqnq8iLQCpopIyxLK+wGZqtpWRNqWsN+Q8r+37qFOYhwpqelcMWAELRvV4pSOzQGYOHUBV5z3zx7dfvPW3dRJrE5KajqXDxhBi8a1ycnN4+/0TL579z4W/r6Bmx9+h8WfDfHUxPhDVasSybgXbuWhoRNJ37PvoG2vvPcdz917FbPGD+b31VtYumoTubl5nHnCMbRr1ZBp4x4AoHJUJVJSMwB4/8XbaFSvJpUiwqlfu0bBdwCjJ8zgwy/mFhmZCn9ObtW0NkMGXsoVA0b6p8NHoKL+74rIR0BXnFzvJuBx4HngExHpA/wFXO1W/xq4AFgNZAI3+3KMUA24BxGRkcBpOHncTcBwAFX9Q0Q2AC3d7UWVnwG85pYvFZGlJRynL84UDxo0bOi3/viiTmIc4HwsvrBrOxb8voFTOjYnJyeXL2csYdp79we1fcGW//E6sUYMF3Vty8Ll66mXVJ2Lz2qHiNDpuMaEibDz7wwS4r2ZWogID+O9F27jv9/M58vpSw7bnr5nHwOe/KDg+ZLPn2DDlp106dCcCV/N48mRUw57Te8H3gSKz+Fu2f439WrFsyX5b8LDw4iNrkLarj0A1E2qzvsv9qXf4++zfvOOiuxq+VRQxFXVXsVsOruIuoo7yCuLUE0pLAcKJhKran+cTidS/Okt6bSXmsx2jzNGVTuraueEhERf21rh9uzdXzCa2bN3P9Pn/cGxzZwZJzN+XUmLRrWoVys+aO0LtkPPz7S5f3Bss7pc0LUts351vjlfvWE7Wdk5B32p5jXDH7ueVeu38fqH04rcHhtdhUoR4QDccNkp/LRoNel79jHr15Vc0q09CfFO36vHVqVBbd/eL9/MXkavC08C4NJuHQrOZ2x0FT4edgdPjpzCvKVry9u1ChXm3rm3tEcoCNUR7jTgWRHpp6r5E4qruj9nAdcD09yUQUNgpQ/l00WkDdA2cN04Mimp6fS+3xmJ5OTmcdV5nenepTUAk6cu+Md/WZayM51/uSO13JxcruzRme6ntCYrO4cBT46ny7XPEFkpnFFDens2nXByu6b0vPAklv+5ueBj/1Mjp1C/dg0A3p00h2Oa1GbUkN7k5uWxct02Bj41HoCV67bxzOgvmTRiAGEiZOfkcv+Ln7BxW1qpx33/858Y/cQNLJj0OGm799DnkXcBuO2aM2jSIJH7b+3B/bc61wZcMWAEO9Iy/NH9MvHS/2FxRsahx/1GcBhwEpAC7AFG4yStRwOdgBzgHlWdLiKViymvArwLtAYWA82BQaVNC+vYqbP+OPdXv/TtaODVQBZI8ScMCHYTQtr+lZ+Ql5lcrjdSm3YdddLUOT7VPaZ2tQUVOC3siITqCBf3G7+exWy+qYj6+4op31vCfowxHmYLkBtjTKDYAuTGGBM4Hoq3FnCNMV5mC5AbY0zAeCjeWsA1xniX70vdhgYLuMYYb/NQxLWAa4zxNJsWZowxAWI5XGOMCQQBH5f6DQkWcI0xHuediGsB1xjjWfkLkHuFBVxjjKd5KN5awDXGeJuNcI0xJkDs0l5jjAkQ74RbC7jGGA8r5x15A84CrjHG0+xKM2OMCRTvxFsLuMYYb/NQvLWAa4zxstC5BbovLOAaYzzLa1eahQW7AcYY809hI1xjjKd5aYRrAdcY42k2LcwYYwLBLnwwxpjA8NqXZhZwjTGeZikFY4wJEBvhGmNMgHgo3lrANcZ4nIcirgVcY4xnCXjq0l5R1WC3ISSJSAqwIdjtKCQB2BHsRoQ4O0clC7Xz00hVE8uzAxH5Bqdfvtihqj3Kc7zysoDrESIyX1U7B7sdoczOUcns/ASfraVgjDEBYgHXGGMCxAKud4wJdgM8wM5Ryez8BJnlcI0xJkBshGuMMQFiAdcYYwLEAq4ficjlIqIi0irYbfE6EckVkcUiskREForIKWV8/RARuc9f7QsGEaklIh+KyFoRWSAiP4vI5RWw3xkiYtPH/MACrn/1AuYAPcu7IxEJL39zPG2vqrZX1XbAQ8BzwW5QMImIAJ8Bs1S1qap2wnmf1Q9uy0xJLOD6iYhEA6cCfXADroh8LCIXFKozVkSuFJFwEXlJRH4VkaUicru7vauITBeRD4Flbtln7mhmuYj0LbSvPiKyyh2dvCkiI9zyRBGZ6O77VxE5NXBnwW9igTRwzrOI/OCOepeJyKX5lUTkERFZKSLfA8cEq7F+0g3IUtXR+QWqukFVh4tIZRF51z0fi0TkLIASyquIyAT3vfcxUCU4XTr62VoK/nMZ8I2qrhKRVBHpCEwArgW+FpFI4GygH05Q3qWqJ4hIFPCjiEx193Mi0EZV17nPb1HVVBGpAvwqIhOBKOAxoCOQDkwDlrj1XwWGqeocEWkIfAsc6+e++0MVEVkMVAbq4AQcgH3A5aq6W0QSgLkiMgXnXPQEOuC8zxcCCwLfbL85DqdPRekPoKrHu+msqSLSsoTyfkCmqrYVkbYl7NeUkwVc/+kFvOL+PsF9/hjwmhtUe+B8HNwrIucCbUXkKrd+HNACyAJ+KRRsAQYVytM1cOvVBmaqaiqAiPwXaOnW6Q60lgMLfMSKSIyqpldsd/1ur6q2BxCRLsA4EWmDs37JsyJyBpAH1ANqAacDk1U1033NlOA0OzBEZCRwGs57ZhMwHEBV/xCRDTjvh9OKKT8DeM0tXyoiSwPfg38GC7h+ICI1cUZgbUREgXBAgQeAGcB5OCPdj/JfAgxU1W8P2U9XYM8hz7sDXVQ1U0Rm4Iz4SlouKcytv7e8/QoVqvqzO5pNBC5wf3ZS1WwRWY9zTsA550er5cCV+U9Utb97TuYDm4t5TUnvk6P5XIUMy+H6x1XAOFVtpKqNVbUBsA5nhDEBuBlnBJYfYL8F+olIJQARaSki1YrYbxyQ5gbbVsDJbvkvwJkiEi8iERT6hwhMBQbkPxGR9hXWyyBx+x4O7MQ5J8lusD0LaORWmwVc7uYnY4CLg9Nav5kGVBaRfoXKqro/ZwHXg/NeAhoCK30sbwO0DUD7/5FshOsfvYDnDymbCFwHDALGAVNUNcvd9hbQGFjofvucgpMDPtQ3wB3uR76VwFwAVd0sIs8C84AtwO/ALvc1g4CR7msicP5x3VEBfQy0/BwuOCO1G1U1V0TGA1+IyHxgMfAHgKoudL8AWoyzzObsYDTaX1RVReQyYJiIPIDzntkDPAh8DowWkWVADnCTqu4XkdeLKR8FvOu+Rxbj/AE3fmCX9h4lRCRaVTPcEe5k4B1VnRzsdhljDrCUwtFjiDsC/A0nffFZkNtjjDmEjXCNMSZAbIRrjDEBYgHXGGMCxAKuMcYEiAVcc0TkwOpdv4nIf0WkaumvKnZfXUXkS/f3S0RkcAl1q4vInUdwjCJXCyuu/JA6YwtdBejLsRqLyG9lbaM5+lnANUcqf/WuNjiXkx40t1ccZX5/qeoUVT10DnNh1YEyB1xjQoEFXFMRZgPN3ZHdCneC/UKggYicK846rQvdkXA0gIj0EJE/RGQOcEX+jkTkJjmw0lktEZkszhq4S8RZA/d5oJk7un7JrXe/HFhp7YlC+yrTamEicpu7nyXirLBWeNTeXURmi7Mi20Vu/SJXeTOmOBZwTbm4F1qcj7t8JE5gG6eqHXCufHoU6K6qHXGu879HRCoDb+Jcbns6zuI7RXkNZ1Gedjirfy0HBgNr3NH1/e7CPy1wVlVrD3QSkTNEJH992A44Af0EH7ozSVVPcI+3AmcVt3yNgTOBC3Gu1qpMoVXe3P3fJiJNfDiO+YeyS3vNkSp8qe1s4G2gLrBBVee65ScDrXGWmwSIBH4GWgHrVPVPABH5AOjL4boBNwCoai6wS0TiD6lzrvtY5D6PxgnAMZR9tbA2IvI0TtoimgNrXQB8oqp5wJ8istbtQ3GrvK3y4VjmH8gCrjlSBcsl5nOD6p7CRcB3qtrrkHrtqbjVqQR4TlXfOOQYdx3BMcYCl6nqEhG5CehaaNuh+1KKX+WtcRmPa/4hLKVg/GkucKqINAcQkaruKlV/AE1EpJlbr1cxr/8BZ3Hs/HxpLM4C6zGF6nwL3FIoN1xPRJI4stXCYoCt4qzadv0h264WkTC3zU1xFg/ydZU3YwAb4Ro/UtUUd6T4kTiLrgM86t4Foy/wlYjswLnvW5sidvF/wBgR6QPkAv3ctXB/dKdd/c/N4x4L/OyOsDOAfx3hamGP4ay4tgEnJ104sK8EZuIsbn6Hqu4TEV9XeTMGsLUUjDEmYCylYIwxAWIB1xhjAsQCrjHGBIgFXGOMCRALuMYYEyAWcI0xJkAs4BpjTID8PwL+I1tFToKfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Best parameters set found on development set:\\n\", clf.best_params_)\n",
    "print(\"\\n Best Score\", clf.best_score_) \n",
    "\n",
    "print(\"\\n Grid scores on development set:\")\n",
    "means = clf.cv_results_['mean_test_score']\n",
    "stds = clf.cv_results_['std_test_score']\n",
    "for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "    print(\"%0.3f (+/-%0.03f) for %r\" % (mean, std, params))\n",
    "\n",
    "print(\"Detailed classification report:\")\n",
    "print(\"The scores are computed on the full evaluation set. \\n\")\n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(classification_report(y_true, y_pred))\n",
    "\n",
    "#Lets print confision matrix\n",
    "labels =['Average','Bad','Good']\n",
    "disp = plot_confusion_matrix(clf, X_test, y_test,\n",
    "                                 display_labels=labels,\n",
    "                                 cmap=plt.cm.Blues)\n",
    "\n",
    "print ('\\nTesting score: ',clf.score (X_test, y_test)) # print the testing score(accuracy)\n",
    "print ('\\nTraining Score: ',clf.score (X_train, y_train)) # print the trainig score(accuracy)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start time 21 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:    5.7s remaining:    8.5s\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   5 | elapsed:    5.7s remaining:    3.8s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 80 ms, sys: 61 ms, total: 141 ms\n",
      "Wall time: 6 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:    6.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:    6.0s finished\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "scoring = ['f1_macro']\n",
    "scores = cross_validate(clf, X, Y, cv=5, scoring=scoring, verbose=10, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "['fit_time', 'score_time', 'test_f1_macro']\n",
      "test_f1_macro [0.47942543 0.46982902 0.47266061 0.46363159 0.47202167]\n"
     ]
    }
   ],
   "source": [
    "print(\"-------------------------------------\")\n",
    "print(sorted(scores.keys()))\n",
    "print('test_f1_macro', scores['test_f1_macro'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K select Percentile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.000210523684764536 0.005160091115494809 0.1361031518624642 0 0]\n",
      " [0.06578947122576177 0.013897588526412541 0.14469914040114612 0 0]\n",
      " [0.005263155277008304 0.010694145453587434 0.11891117478510028 0 1]\n",
      " ...\n",
      " [0.05263157645429362 0.859810398981424 0.1346704871060172 0 0]\n",
      " [0.000988368423653601 0.005792679656356735 0.1017191977077364 1 0]\n",
      " [0.17105262939750693 0.057828596847734885 0.13753581661891118 0 0]] ['Bad' 'Bad' 'Bad' ... 'Average' 'Bad' 'Average']\n",
      "4800 4800 1200 1200\n"
     ]
    }
   ],
   "source": [
    "#Load Kbest data\n",
    "arr2= np.load('data_select_percentile.npy', allow_pickle= True)\n",
    "X= arr2[:,0:-1]\n",
    "Y= arr2[:,-1]\n",
    "\n",
    "\n",
    "### Split arrays or matrices into random train and test subsets - 20% for Test, 80% to train\n",
    "X_train, X_test, y_train, y_test= train_test_split(X, Y, test_size=0.20,  random_state = 200)\n",
    "\n",
    "print(X_train,y_train)\n",
    "print(len(X_train),len(y_train),len(X_test),len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.17 s, sys: 0 ns, total: 6.17 s\n",
      "Wall time: 6.17 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score=nan,\n",
       "             estimator=LinearSVC(C=1.0, class_weight=None, dual=True,\n",
       "                                 fit_intercept=True, intercept_scaling=1,\n",
       "                                 loss='squared_hinge', max_iter=1000,\n",
       "                                 multi_class='ovr', penalty='l2',\n",
       "                                 random_state=None, tol=0.0001, verbose=0),\n",
       "             iid='deprecated', n_jobs=None,\n",
       "             param_grid=[{'C': [0.1, 1, 10, 100, 1000]}],\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1_macro', verbose=0)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "tuned_parameters = [{'C': [0.1,1, 10, 100, 1000]}]\n",
    "clf = GridSearchCV(LinearSVC(), tuned_parameters, scoring='f1_macro')\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters set found on development set:\n",
      " {'C': 100}\n",
      "\n",
      " Best Score 0.5757342325404655\n",
      "\n",
      " Grid scores on development set:\n",
      "0.543 (+/-0.012) for {'C': 0.1}\n",
      "0.564 (+/-0.013) for {'C': 1}\n",
      "0.567 (+/-0.012) for {'C': 10}\n",
      "0.576 (+/-0.022) for {'C': 100}\n",
      "0.422 (+/-0.082) for {'C': 1000}\n",
      "Detailed classification report:\n",
      "The scores are computed on the full evaluation set. \n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Average       0.60      0.26      0.36       405\n",
      "         Bad       0.57      0.78      0.66       392\n",
      "        Good       0.58      0.70      0.63       403\n",
      "\n",
      "    accuracy                           0.58      1200\n",
      "   macro avg       0.58      0.58      0.55      1200\n",
      "weighted avg       0.58      0.58      0.55      1200\n",
      "\n",
      "\n",
      "Testing score:  0.550562115309828\n",
      "\n",
      "Training Score:  0.5687583029163399\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEGCAYAAAApAy29AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3wV1bbA8d9KKElIAgFCB5EmAtIVCyqKIFjAcrmA2LmCXMReQK9Pve9ang3Fjg8pKiCoKA+VIgiIovTeizSRTggQSpL1/pgJBkiZJKcNru/9zCc5e/bZs+fc42Jnz541oqoYY4wJvqhwd8AYY/4qLOAaY0yIWMA1xpgQsYBrjDEhYgHXGGNCpFi4OxCpyiSV00pVq4e7GxHrcHpGuLsQ8dIzw92DyLZ/xzYOp+yVorQRnXiWanqap7qatmuSqnYoyvGKygJuLipVrc7gL6eFuxsRa+GOlHB3IeLtO5we7i5EtMH9bipyG5qeRslz/u6p7pFF75Qv8gGLyKYUjDE+JiBR3rb8WhKJEZE5IrJYRJaLyHNu+dki8quIrBWRz0SkhFte0n29zt1fM79jWMA1xviXAFHR3rb8HQWuVNUmQFOgg4hcCPwPMFBV6wL7gJ5u/Z7APlWtAwx06+XJAq4xxt9EvG35UMdB92Vxd1PgSuBzt3w4cIP7e2f3Ne7+tiJ5H8gCrjHGxwo0pVBeROZl23qd1ppItIgsAnYCU4D1wH5VzZqQ3wpUdX+vCmwBcPenAOXy6q1dNDPG+JuH0atrt6q2zKuCqmYATUWkDDAOODenallHzmNfjmyEa4zxLyFgF82yU9X9wHTgQqCMiGQNTqsBv7u/bwWqA7j7SwN782rXAq4xxsc8zt96GAWLSLI7skVEYoGrgJXAD8Df3Gp3AF+7v493X+Pun6b5pF+0KQVjjL95W4HgRWVguIhE4wxGx6jqBBFZAYwWkf8AC4Ehbv0hwMcisg5nZNstvwNYwDXG+JgUeLogN6q6BGiWQ/kG4IIcyo8AXQpyDAu4xhj/Egpy0SzsLOAaY/wtQCPcULCAa4zxscBNKYSCBVxjjH8JEB2wi2ZBZwHXGONvNodrjDGhYFMKxhgTOjbCNcaYELERrjHGhIDH23YjhQVcY4y/Be7W3qCzgGuM8TG7aGaMMaFjUwrGGBMCWflwfcICrjHGx2xKwRhjQscumhljTIjYHK4xxoSA2JSCMcaEjo1wjTEmNMQCrjHGBJ/zhB0LuMYYE3wiSJQF3L+8194bx68L1lAmsRSDX7vvtP2qynvDvmXOwrXElCzOI31upG6tKkU65oGDh3nhjTHs2LWfislleOrBriTExzLtx8WMGT8LgJiYEvTreT21a1Yq0rGKauynE1m1fAPxCXE8NODOXOtt2fQH774+klvuvI7zmtUr0jEPH0pj5LAJ7Nt7gKSyidxy1/XExcWwcO5KZkydA0CJEsW5oetVVKlaoUjHCoRvx05h/aqNxMXH0fOhW3Oss3n9VqZOmEFGRiZxpWK5pfffinTM9PR0vhkzmT+27SQ2LobO3a+hdNlENq7dxIyJP5ORnkF0sWiu6Nias+pUL9KxAsVPI9ygXt4TkRtFREWkfjCPE4naX96M5wfcluv+uYvWsu2PPQx98wEeuKcTbw35P89tL16+kVff/fK08jFf/UizRrUY+uaDNGtUi8++/hGAihWSeOWZu3n/lb70uOly3vzw64KfUIC1aNWIu/vcnGedzMxMvhs/k3rn1ixQ2+vXbmHMJxNPK5/+/Rzq1KvBY0/3pE69GsyY4gTZsuUS6XV/Vx7sfwdtO1zEuNFTCnS8YDmvRQO63H1DrvuPpB1l8tc/cPMd1/OPh2+jc49rPLedsvcAIz/4/LTyJXOXExNbkt6P3UnL1s2YPtH5hzouLpab77ieng/dyrVd2jFhzKSCn1CQiIinLRIEez1Fd2AW0K2oDYmIf1Y3A+c1qElCfGyu+2fPXcVVlzVFRDi3XnUOHTrCnn2pAIwdP4t+A97n3sfeYcSYaZ6POXveKq66vBkAV13ejNlzVwLQ8JwaJ/pSv251du85UNjTCphadaoRGxeTZ52fZyzkvCZ1KRUfd1L5jKlzefvVT3jjpeFM+fYnz8dcsXQ9zS9oCEDzCxqyfOk6AM6qVZU4ty/Va1YmZf/BgpxK0FSvVZXY2Nw/oxWLVlGvYW0SyyQCnPQ5LV+4ihFvj2bom58y8cupZGZmejrm2hUbaNS8AQD1G9Vl07otqCoVq1YgITEegPIVy5GenkF6enphTy2gLOACIhIPXAL0xA24IvKZiFyTrc4wEblZRKJF5BURmSsiS0Skt7u/jYj8ICIjgaVu2VciMl9ElotIr2xt9RSRNSIyXUQ+FJG33fJkEfnCbXuuiFwSrHMuiN37DpBcrvSJ1+XLJbJn7wHmL17Htj/2MOiF3rz7P31Yu/F3lq74zVOb+1IOUS4pAYBySQnsP3DotDoTf5jP+U3rBuQcgillfyrLl6yjVesmJ5WvWfkbe3bto+8jPbj/8dvZtmUnG9Zt9dTmwdTDJJZ2gkZi6XgOph4+rc682UsLPKIOl72793Mk7SgjP/icYW+NYtl85x/Y3Tv3snLxGnr06cJdD/QgKkpYsXC1pzYPHjhEQhnnM4qKjqJkTEnSDh85qc7qZeuoWCWZYsUiYEZSCrBFgGB+YjcAE1V1jYjsFZHmwGigK/CtiJQA2gJ9cIJyiqqeLyIlgZ9EZLLbzgVAI1Xd6L6+W1X3ikgsMFdEvgBKAk8DzYFUYBqw2K3/JjBQVWeJSA1gEnBuEM/bG9XTikSE+UvWsWDJev75xHsApB05xrY/9nBeg5rc/9QHHD+eQdqRY6QeTKPP4+8C0POWdrT0EEQXLdvApGkLeP3f/wjsuQTBhC+n07HTpURFnTwmWLv6N9as2sSglz8G4NjR4+zZtY9adarxzmufkp6ewbGjxzl8+Ahv/s8IADp2usxTEF2/ZjNzf1nGvQ8W+Q+ykNDMTP7YtpNu99xE+vF0Pnn3M6rUqMSmdVvYsW0nI94eDUD68XTiSjmj3y9HTCBlXwoZGZkc2J/K0Dc/BaDFJU1p3LIhmtP3Mtvvu3bsYcZ3P/H3nrlPdYSSEDmjVy+CGXC7A2+4v492Xz8NDHKDagdgpqqmiUh7oLGIZM34lwbqAseAOdmCLcD9InKj+3t1t14lYIaq7gUQkbFA1hWWq4AG2f5PSRSRBFVNPbXD7oi5F0DFKtWKdPL5KV+2NLv2pJx4vXvPAcomJaAKXTtfyrXtzj/tPYOe7w04c7hTZizk0X/edNL+pNKl2LMvlXJJCezZl0qZxFIn9m3Y9AdvDP6a//S/jcSEk/9Ej0RbN//ByOHfAHD4YBqrV2wgKlpQhSvaXUCrS5qc9p6+j/QAnDnc+b8u5++3djhpf3xCHAdSDpJYOp4DKQeJz/Y5bN+2iy9GTeauPjdRqlTuU0GRJKF0PLFxsZQoUZwSJYpT7eyq7Ny+G1Rp1OJcLu9w+h9zN91+HeDM4X4zdvJpF9kSSseTuv8giaUTyMzI5OiRo8S40y0HUlIZ9/EErv17e5LKlQn+CXp06j/KkSwoPRWRcsCVwP+KyG/AYzgj26PAdOBq9/XorLcA/VS1qbudrapZI9xD2dptgxNAL1LVJsBCIIa8/2CIcutntV01p2ALoKqDVbWlqrYsk1SuMKfu2YUtz+H7mYtQVVau2UJcXAzlkhJo2aQOk6YvIO3IUQB27z3A/hRvc4oXtqzP9zMWAvD9jIVc1NK5Vrlz937+/dpoHut7M9WqlA/OCQXYE8/eQ393a9S0Hjd0uYqGjetSr35N5v2yjKNHjwHO1ENOUwM5adCoNgvmLAdgwZzlNDivNgD79x7gkyHj6XpbR5IrlA3OCQVBnQa12frbNjIzMjl+7Djbt+ygXIUkzqpTndVL13LooPO5pB0+Qso+b/P2dRvUYtmCFQCsWraWGrWrIyIcSTvK50PHc/nVF1OtZtFW0wSan+ZwgzXC/RswQlV7ZxWIyAygNU6Q/QfQErjT3T0J6CMi01T1uIjUA7bl0G5pYJ+qHnZXPlzols8BBopIEs6Uws24c77AZOA+4BW3H01VdVHAzjQXL745liUrNpKSepgefV7lti5XkJ7hXLi4rt35XNCsHnMXruWuB96gZAlnWRhAiyZ12LxtFw/+60MAYmNK8Ph9f6NM6VwPdULXzpfy/BufMfGHBVQoX5qnHuoKwKefTyf14GHeHjIBgOjoKN5+8d4gnLV3o4ZNYMO6rRw6mMYLT39Au2suJsP9fC5sffroNUu9c2uyc8de3n19FAAlSxan623XnDRazc3l7S5g5NAJzP1lGWWSEulxlzPa+37ibA4dSuOrsVMBZ8TU77Gcl2GF0vhR37F5w1bSDh3hnReG0LpdKzLdz6jZhY0pX6EsZ9eryUdvfoqI0Pj8hiRXcv5BvbT9xYwZMg5VJSo6mnad21A6KTHfYzZu2ZAJYybxwSvDiI2NoVP3jgAs+Hkx+/fs5+dpc/h5mrO64+89bzztgmbIRdD8rBeS05xNkRsVmQ68pKoTs5XdjzN3ej/wBzBeVe9y90UB/wGux/n4duHMATcDHlXV69x6JYGvgKrAaiAZeFZVp7vTAY8CvwMrgb2q+pSIlAfecY9dDGcaI99oU79RUx38pfcVAn81C3ek5F/pL27f4ci4ih+pBve7id/XLC1SuCxWvpaWue4FT3X3DO8+X1VbFuV4RRWUEa6qtsmhbFC2l+VO2ZcJPOlu2U13t6x6R4GOuRx2pKoOFpFiwDickS2quhtn+sIYc4bx20Uz/8w25+9ZEVkELAM24oyEjTFnOIkST1u+7YhUd5ehrnSXnT7glj8rIttEZJG7ZV/aOkBE1onIahG5Or9jRMBCusBQ1UfD3QdjTIhJQG/tTQceUdUFIpIAzBeRrNsOB6rqqycdWqQBzj0GDYEqwPciUk9VM3I7wJk0wjXG/AUFapWCqm5X1QXu76k414Kq5vGWzsBoVT3qLl1dh3PfQK4s4BpjfK0AAbe8iMzLtvXKo82aOBftf3WL7nPvgv3IXQ0FTjDeku1tW8k7QFvANcb4V9ZFM48Bd3fWOnt3G5xjm05agi+AB1X1APAeUBtoCmwHXjtx+NPluezLAq4xxt8CmEtBRIrjBNtPVfVLAFXdoaoZ7mqqD/lz2mArzt2uWarhLEvNlQVcY4x/iXOjipct36acYfAQYKWqvp6tvHK2ajfirIQCGA90E5GSInI2TpqBOXkd44xZpWCM+WsK4CqFS4DbgKXuElNw7g3oLiJNcaYLfgN6A6jqchEZA6zAWeHQN68VCmAB1xjjdwGKt6o6K5fWvs3jPc8Dz3s9hgVcY4yv+elOMwu4xhjfiqRMYF5YwDXG+JoFXGOMCRF7TLoxxoSIjXCNMSYUApu8Jugs4BpjfEsAH8VbC7jGGD+zVQrGGBMyUXbRzBhjQkBsSsEYY0JCsBGuMcaEjI1wjTEmROyimTHGhILN4RpjTGgI4im5eKSwgGuM8TUb4RpjTIjYHK4xxoSCzeEaY0xoOLkU/BNxLeAaY3zNR/HWAq4xxt/sTjNjjAkFy4d7ZogrWYzmZ5UJdzci1tVd/yvcXYh4W398I9xdiGhfxZcochuWD9cYY0LG8uEaY0zI+CjeWsA1xviY2EUzY4wJCVuHa4wxIWQB1xhjQsRH8dYCrjHG32yEa4wxoeCz5DX+ydxrjDGncBKQe9vybUukuoj8ICIrRWS5iDzglpcVkSkistb9meSWi4gMEpF1IrJERJrndwwLuMYYX4sS8bR5kA48oqrnAhcCfUWkAdAfmKqqdYGp7muAjkBdd+sFvJdvXwt+esYYEzlEvG35UdXtqrrA/T0VWAlUBToDw91qw4Eb3N87AyPU8QtQRkQq53UMm8M1xviWFCx5TXkRmZft9WBVHZxzu1ITaAb8ClRU1e3gBGURqeBWqwpsyfa2rW7Z9tw6kGvAFZHEvHquqgfy2m+MMaFQgBvNdqtqy/wqiUg88AXwoKoeyCOg57RD82o7rxHucvfN2RvNeq1AjbwaNsaYUAjkrb0iUhwn2H6qql+6xTtEpLI7uq0M7HTLtwLVs729GvB7Xu3nGnBVtXpu+4wxJhIIzkqFgLTlDGWHACtV9fVsu8YDdwAvuT+/zlZ+n4iMBloBKVlTD7nxNIcrIt2AWqr6gohUw5nTmF+gszHGmCAI4AD3EuA2YKmILHLLnsQJtGNEpCewGeji7vsWuAZYBxwG7srvAPkGXBF5GygOXAa84Db8PnB+Qc7EGGMCTgKXD1dVZ5HzvCxA2xzqK9C3IMfwMsK9WFWbi8hC9yB7RaToqdqNMSYA/HSnmZeAe1xEonCvvolIOSAzqL0yxhgPBLze1BARvATcd3Cu2iWLyHPA34HngtorY4zx6IxKQK6qI0RkPnCVW9RFVZcFt1vGGJM/r3eRRQqvd5pFA8dxphXsdmBjTMTw05RCvsFTRJ4CRgFVcBb2jhSRAcHumDHGeCEet0jgZYR7K9BCVQ8DiMjzwHzgxWB2zBhjvDjTEpBvOqVeMWBDcLpjjDHeOasUwt0L7/JKXjMQZ872MLBcRCa5r9sDs0LTPWOMyYN4Sy4eKfIa4WatRFgOfJOt/JfgdccYYwrmjJhSUNUhoeyIMcYU1BkzpZBFRGoDzwMNgJisclWtF8R+GWOMJ34a4XpZUzsMGIrzj0lHYAwwOoh9MsYYz/y0LMxLwI1T1UkAqrpeVf8FXBHcbhljTP5EIDpKPG2RwMuysKNuYt71InIvsA2okM97TBEcOXqcTn3e5NixdNIzMrn+yqY8cc81zJy7mmff+ppMVUrFluStp3tQq3pyuLtbKCVLFOObwQ9SsngxootFM37qQl4a/O1JdS5uVpsXHv4bDetUoedTQxk/bVEurXlXJjGOj164mxqVy7J5+17uGjCElNQ0unRoyQO3twPgUNpRHnnpM5at3Vbk44VTRkYm19zzGpXKl2b4y72YNW8N/3l3PJmaSanYkrz+5C2cXc2f35/szrQphYeAeOB+nAS99wB3B7NTeRGRDBFZJCKLRWSBiFxcwPc/KyKPBqt/gVCyRDG+fLsf0z/pzw8fP8G02SuZt2wjj708hvefu53pHz/Bze1b8PrQSeHuaqEdPZZO5z6DuLTHS1x2y4u0vagBLRvVPKnOlj/20fe5j/l80rycG8nDJc3r8s4zt55W/tAd7Zg5dzUtb/43M+eu5qE72gOw6fc9XNv7DVrf8iKvDJnIwCe7F+q8IsmQsTOoc1bFE68HvDaWt/7rViYPfZwb2rVg0PDJYexd4ATqqb2hkG/AVdVfVTVVVTer6m2q2klVfwpF53KRpqpNVbUJMIAz8I43ESE+riQAx9MzOJ6egeAkWk49dASAA4fSqJRcOpzdLLJDaccAKF4smuLFonHyOf9py/a9LF/3O5l6+nP5+t3alqnDH2PWyAH073WN52N2vLwxoyb8CsCoCb9yTZvGAMxZspGU1DQA5i7dSJUKZQp1TpHi9537mTp7Bbdcd+GJMhFOfH9SD6ZRsby/vz/gPF4nSrxtkSCvGx/GkccTKFX1pqD0qGASgX1w4kmbXwNJOE+o+Jeqfu3uewq4HeeRxrtwbk2OaBkZmbS98xU2bt1Fz5svpUWjmgx8sjvdH36fmJLFSSgVw8QhD4e7m0USFSVM//gJzq6WzJCxM5m/fJOn913Rqj61alSg7R2vICKMeq03Fzerzc8L1+f73gplE9ixx3ng9I49B0hOSjitzm2dL+b7n1cU7GQizLODxvHUPztx8PCRE2WvPNGN2x8f7Hx/4mIY/8FDYexhgETQ6NWLvOZw3w5ZLwom1n3eUAxQGbjSLT8C3Og+1rg88IuIjAeaA91wnjFfDFhALgFXRHoBvQCqVw/vQ4mjo6OY/vETpKQe5o4n/peV63/ng1E/MOr1e2nRqCZvfzKVp98YxxtP3RLWfhZFZqZyWY+XSIyP5ZNX7uHc2pVZuT7PZ/ABcMWF53Jlq/rM/LQ/AKViS1KregV+XrieKUMfpWSJYpSKLUlSYtyJOs++9TXTflmZb9utW9Tl1k4X0fGegUU7uTD6/qfllE+Kp/E51fl54doT5R+OmcGIl3vRvGFN3hs5jefe+opX+3cLY08Dw09zuHnd+DA1lB0pgDRVbQogIhcBI0SkEc7KjxdE5DKcJ1JUBSoClwLjsiXfGZ9bw6o6GBgM0LxFyzyfLx8qpRPiuKR5XabOXsnyddto4c5z3nBVM7o++F54OxcgBw6mMWv+Wtpe1MBTwBWBgcMmM2zc6TNb7e56FXDmcG+5vhV9n/vkpP0796ZSsVwiO/YcoGK5RHbtSz2xr2GdKgz61y10eeA99qUcKuJZhc/cpRuY/NMypv2ygqPH0kk9dITbHxvM+s07aN6wJgCd2jbj1kfeD29HA0CAaB8FXF/ntlXV2UB5IBno4f5s4QbkHfx5o0ZEBE+vdu9LJSX1MABpR44xY+5q6tWsyIGDR1i/eScA0+espm7NSuHsZpGUKxNPYnwsADEli9PmgnNY+9sOT++dNnslPTpdRKlY59F6lZNLUz4p3tN7J85cSvfrWgHQ/bpWfDdjCQDVKiYx4uV7uPeZESc+Y78acO/1zPvyOX4Z+wzvPHs7lzSvy0cv9uTAoSNscM9t5tzV1KlZMZ+W/CFKvG2RwGsC8ogkIvVxkqPvAUoDO1X1uIhcAZzlVpsJDBORl3DO93rgg3D016sduw9w339/QmaGkqlK57ZNad+6Ea8P6MZdA4YQJULphDje/Jd/pxMqlU/k3WdvIzoqiqgoYdz3C5g0axkDel/LopWb+W7mUpo1qMHHL99DmcQ4OrQ+j/69r+Xirs/zw6+rqHd2JSZ/5Cw2OXj4KL3/azi79x3M97gDh09h6It3c2uni9i6Yx939nfuYH/sHx0pW7oUrz7RFYD09EyuvOPl4H0AIVasWDQvP96Ve54e6n5/YnltgP9XYkDkBFMv5NQrw7lWFCmpqkeD3B8v/cgAlma9BJ5U1W/cedv/w7lgtghnCVtHVf0t20WzTcBWYIWqvprXcZq3aKkzf54TrNPwveQL7w93FyLe1h/fCHcXIlrby1qxaMH8IoXLSnUbaY/Xv/BU9/VO9eerasuiHK+ovORSuAAYgjOCrCEiTYB/qGq/YHcuJ6oanUv5buCiXPY9j5MPwhhzhvHTCNfLHO4g4DqcP9tR1cXYrb3GmAjhpxsfvMzhRqnqplOWXmQEqT/GGOOZAMUiJZp64CXgbnGnFVREooF+wJrgdssYY7zxUbz1FHD74Ewr1MBZavW9W2aMMWElEXTbrhf5BlxV3Ylzp5YxxkQcH8VbT6sUPiSHGwdUtVdQemSMMQVwpq1S+B6Y6m4/4eTCDft6XGOMEQKXgFxEPhKRnSKyLFvZsyKyzU0Ju0hErsm2b4CIrBOR1SJytZf+eplS+OyUTn0MTPHSuDHGBFVgb9sdhpO0a8Qp5QNPvVFKRBrgTLU2BKoA34tIPVXNcwVXYXIpnM2ft80aY0xYicf/5UdVZwJ7PR62MzBaVY+q6kZgHXBBfm/KN+CKyD4R2etu+3FGt0967JQxxgRN1mPSg5y85j4RWeJOOSS5ZVVx8mtn2eqW5SnPgOs+y6wJThauZCBJVWup6pjC9dsYYwKrAAG3vIjMy7Z5ufD/HlAbaApsB15zy3MK4fkmpslzDldVVUTGqWoLDx0zxpiQK0AC8t0FTV6jqidyhrortia4L7cC1bNVrQb8nl97XuZw54hI84J00hhjQsF5TLq3rXDtS+VsL28EslYwjAe6iUhJETkbqAvkm14wr2eaFVPVdKA1cI+IrAcO4QylVVUtCBtjwi5Qd5qJyCigDc7Uw1bgGaCNiDTFmS74DegNoKrLRWQMsAJIB/rmt0IB8p5SmIPzPLAbinAOxhgTNFkXzQJBVXPKyD4kj/oFTvuaV8AVt9H8H4VqjDFhcqbc2pssIrk+h1tVXw9Cf4wxpgCEKA9rbCNFXgE3Gogn5+UPxhgTdsKZM8Ldrqr/DllPjDGmoASK+Sh7Tb5zuMYYE6nOpBFu25D1whhjCumMSECuql6TOBhjTNj4KN56esSOMcZEJKFwKQ/DxQKuMca/5AyZUjDGmEjn3GlmAdcYY0LCP+HWAq4xxud8NMC1gGuM8TMpSD7csLOAa4zxLVulYIwxIWQXzc4AAhQrbJr4v4Cfv3oh3F2IeNX+9la4uxDRjq7fWfRGpECP2Ak7C7jGGN+yKQVjjAkhG+EaY0yI+CfcWsA1xviYANE2wjXGmNDwUby1gGuM8TNBfDSpYAHXGONrNsI1xpgQcJaF+SfiWsA1xviX2AjXGGNCxm7tNcaYEHASkIe7F95ZwDXG+JqtUjDGmBDx0YyCBVxjjL/ZCNcYY0LAb3O4fspsZowxJxMhyuOWf1PykYjsFJFl2crKisgUEVnr/kxyy0VEBonIOhFZIiLNvXTXAq4xxtfE4+bBMKDDKWX9gamqWheY6r4G6AjUdbdewHteDmAB1xjjW86UQmBGuKo6E9h7SnFnYLj7+3DghmzlI9TxC1BGRCrndwwLuMYYXyvACLe8iMzLtvXy0HxFVd0O4P6s4JZXBbZkq7fVLcuTXTQzxvib94tmu1W1ZRCPqvm9yQKuMcbXgnxr7w4Rqayq290pg6wnX24FqmerVw34Pb/GbErBGONrAbxolpPxwB3u73cAX2crv91drXAhkJI19ZAXG+EaY/wtQANcERkFtMGZ690KPAO8BIwRkZ7AZqCLW/1b4BpgHXAYuMvLMSzgGmN8yxm9Bibiqmr3XHa1zaGuAn0LegwLuMYY/7J8uMYYEzo+ircWcI0xfiaIj4a4FnCNMb7mo3hrAdcY419FXPIVchZwjTH+5qOIawHXGONrloDcFMmRo8e5ttcbHD2eTkZ6Bp3aNmNA72vp99+fsnDlZlSVOjUq8M4ztxEfVzLc3Q2L0eNn8X9T5oFA7bMq8VS/m1m6ajNvD/uO4+np1K9dlQH33USx6Ohwd7VQqpaP572HO1AhKY7MTBg+aSkfjF94Up3EuBJ88GhHqiUnEB0VxcYGuNIAABAFSURBVNvj5jHy+xVFOm6Z+JJ89MS11KiYyOYdB7jrpW9IOXSULm3q88DNThqCQ0eO88i7U1m2cXeRjhUofprDjdhbe0WkooiMFJENIjJfRGaLyI0BaHe6iAQqgUVQlCxRjK/fu59ZIwcwc+QAps5ewdylG3n+oZuYNXIAP416kmqVkvhwzIxwdzUsdu1JYeyE2Xz0al8+HfQgmRmZTJm5mP+8+Tn/fqQbnw56kErJZfhu2sL8G4tQ6RnKv4bM5MI+I2j/6Cj+cW0Tzqle9qQ6/7i2Cas37+HSfp9w/YCx/Kfn5RQv5u0/6UvOq8Y7D7Y/rfyhLhcwc/EWWvYaxszFW3ioy/kAbPojhWv7j6V1v094ZfSvDLzvqqKfZCC463C9bJEgIgOuOOs8vgJmqmotVW0BdMNJEHHGE5ETI9fj6RkcT89AREiMjwVAVUk7etxXy2ECLSMjk6PHjpOekcGRY8eJiSlB8eLR1KhaHoDzm9Rh+uxl+bQSuXbsO8SS9U6elINpx1mzZS+Vy8WfVEeB+NgSAJSKLc6+1COkZ2QC0O+mFkx9vTuz3rqV/rdc5Pm4HVvVYtRUZ5Q8auoKrrmwNgBzVm0n5dBRAOau2k6V8glFOr9AEo//iwQRGXCBK4Fjqvp+VoGqblLVt0QkRkSGishSEVkoIlcA5FEeKyKj3cdgfAbEhueUCiYjI5NLb3mReu3706ZVfVo2qglA3+c+5pwOT7L2tx306np5eDsZJsnlStP9htbceM/LdLrrReLjYmh7yXmkZ2Syct1WAH6YvYwdu1PC3NPAqF4hkca1kpm/+o+Tyj+csIh61cuyckQvfnr7NgYMno4qXNGsBrWqJNH24VFcev8nNK1TgYsb5puqFYAKZeLYse8Q4AT95DJxp9W5rX0jvp+3segnFgCCv0a4kTqH2xBYkMu+vgCqep6I1Acmi0i9PMr7AIdVtbGINM6j3YgSHR3FjyMHkJJ6mFsf+5AV636nQZ0qvPPMbWRkZPL4K2MZN3k+PTp5H72cKQ4cTOPHOSv5/INHSSgVy1Mvj2TSjEX8+5FuDBryDcfSM7igaR2ioyN1POFdqZjijHjyOgZ8OIPUtGMn7buyeU2WbthFpyc/5+zKpRn33zczu982rmh2Flc2q8HMQT3cNkpQq0oZfl6+jSmvdaNk8WhKxZQgKSHmRJ1nh81i2oJN+fan9XnVuLV9Qzo+PibwJ1tIERJLPYnUgHsSEXkHaA0cw8lD+RaAqq4SkU1APXd/TuWXAYPc8iUisiSP4/TCeT4R1WvUCNr5FETphDhat6jL1NkraFCnCuAE45vaNWfQJ9//JQPuvMXrqFIhiaTSzp/YbS5qyNJVm+nQphnvvdgbgF8XrmXLtsi4qFNYxaKjGP7kdYydvooJs9edtr/HVQ144/N5AGzcnsKmHSnUrZ6EiDBw7FyGTVx62nvaPTIacOZwb2nbgL5vTD5p/879h6mYVIod+w5RMakUu/YfPrGvYc3yDLq/HV2eGce+1COBPNWi8VHEjdQhwHLgxFMwVbUvTsaeZHL/ePP62PPNxO4eZ7CqtlTVlsnlk732NeB270slJdX5oqcdOcb0Oaupc1ZFNmzZldVPJv64lHpnVQxbH8OpYnIZlq/ZwpGjx1BV5i1ZT81qyezdfxCAY8fT+eTLGdzQoVWYe1o0bz3QjjVb9vLuVzn/UbZ1VyqXNXFyYCeXiaNOtbL89kcK0xb8Ro92DSkVUxyAyuVKUb60t5m0ib9uoHvbBgB0b9uA737dAEC15ARGPHk99742kfW/7y/qqQVUoJ5pFgqROsKdBrwgIn1UNetpmFmTSTOBHsA0d8qgBrDaQ/kPItIIaBy60yicP3Yf4J/PfkxGZiaZmcqNVzXn6tYN6XjPG6QeSkMVGtWtymv9u4a7q2HRsF51rri4EXc+/DbR0VHUO7sKna++gMGfTuGneavQTOXGDq1o2bh2uLtaaBc2qEK3KxuwfOOuE3/2//eIn6iWnAjA0O+W8MroX3nnwav56e3bEIHnhv7I3gNH+GHhZupVL8fkV7sBcPDIMXq/OpHdKWn5Hnfg53MZ2v9abm3fkK27UrnzxQkAPNatFWUTY3j1n1cCziqKKx8aGYxTL7DICKXeiJPWMfK4j7MYCLQCdgGHgPdxMq6/D7QA0oGHVfUHEYnJpTwWGAo0ABYBdYD7VXVeXsdv0aKl/vRrnlX+0lZuOxDuLkS8i3sPCXcXItrR2a+TmbKlSPGyUZPm+uXkWZ7qnlOp1PwAPtOsUCJ1hJv1hMxuuey+M4f6R3IpT8ujHWOMjwUyAXkoRGzANcaYfEXQki8vLOAaY3zNR/HWAq4xxs8sAbkxxoSMj+KtBVxjjH9ZAnJjjAklH0VcC7jGGF+zZWHGGBMiNodrjDGhIBBlAdcYY0LFPxHXAq4xxreyEpD7hQVcY4yv+SjeWsA1xvibjXCNMSZE7NZeY4wJkUCGWxH5DUgFMoB0VW0pImWBz4CawG/A31V1X2Haj9RH7BhjTL68PrG3gIPgK1S1abZk5f2BqapaF5jqvi4UC7jGGF8Tj/8rgs7AcPf34cANhW3IAq4xxt/E4wblRWRetq1XDq0pMFlE5mfbX9F9Ak3Wk2gqFLarNodrjPG1Aoxdd3t4ptklqvq7iFQApojIqqL07VQWcI0xPhbYR6Cr6u/uz50iMg64ANghIpVVdbv7cNudhW3fphSMMb6VdadZIC6aiUgpEUnI+h1oDywDxgN3uNXuwHlyeKHYCNcYYxwVgXHuut5iwEhVnSgic4ExItIT2Ax0KewBLOAaY3wtUDMKqroBaJJD+R6gbSCOYQHXGONrloDcGGNCoeA3NYSVBVxjjG9ZekZjjAkhm1IwxpgQsRGuMcaEiI/irQVcY4zP+SjiWsA1xviWQEBv7Q02UdVw9yEiicguYFO4+5FNeWB3uDsR4ewzylukfT5nqWpyURoQkYk45+XFblXtUJTjFZUFXJ8QkXkeMh39pdlnlDf7fMLPktcYY0yIWMA1xpgQsYDrH4PD3QEfsM8ob/b5hJnN4RpjTIjYCNcYY0LEAq4xxoSIBdwgEpEbRURFpH64++J3IpIhIotEZLGILBCRiwv4/mdF5NFg9S8cRKSiiIwUkQ3uU2Zni8iNAWh3uojY8rEgsIAbXN2BWUC3ojYkItFF746vpalqU1VtAgwAXgx3h8JJnOfAfAXMVNVaqtoC53tWLbw9M3mxgBskIhIPXAL0xA24IvKZiFyTrc4wEblZRKJF5BURmSsiS0Skt7u/jYj8ICIjgaVu2VfuaGa5iPTK1lZPEVnjjk4+FJG33fJkEfnCbXuuiFwSuk8haBKBfeB8ziIy1R31LhWRzlmVROQpEVktIt8D54Srs0FyJXBMVd/PKlDVTar6lojEiMhQ9/NYKCJXAORRHisio93v3mdAbHhO6cxnuRSC5wZgoqquEZG9ItIcGA10Bb4VkRI4z0nqgxOUU1T1fBEpCfwkIpPddi4AGqnqRvf13aq6V0Rigbki8gVQEngaaA6kAtOAxW79N4GBqjpLRGoAk4Bzg3zuwRArIouAGKAyTsABOALcqKoHRKQ88IuIjMf5LLoBzXC+5wuA+aHvdtA0xDmnnPQFUNXz3OmsySJSL4/yPsBhVW0sIo3zaNcUkQXc4OkOvOH+Ptp9/TQwyA2qHXD+HEwTkfZAYxH5m1u/NFAXOAbMyRZsAe7PNk9X3a1XCZihqnsBRGQsUM+tcxXQQP5M8JEoIgmqmhrY0w26NFVtCiAiFwEjRKQRTv6SF0TkMiATqIrz9NVLgXGqeth9z/jwdDs0ROQdoDXOd2Yr8BaAqq4SkU0434fWuZRfBgxyy5eIyJLQn8FfgwXcIBCRcjgjsEYiokA0oMDjwHTgapyR7qistwD9VHXSKe20AQ6d8voq4CJVPSwi03FGfHmlS4py66cV9bwiharOdkezycA17s8WqnpcRH7D+UzA+czPVMuBm7NeqGpf9zOZB2zL5T15fU/O5M8qYtgcbnD8DRihqmepak1VrQ5sxBlhjAbuwhmBZQXYSUAfESkOICL1RKRUDu2WBva5wbY+cKFbPge4XESSRKQY2f5DBCYD92W9EJGmATvLMHHPPRrYg/OZ7HSD7RXAWW61mcCN7vxkAnB9eHobNNOAGBHpk60szv05E+gBzncJqAGs9ljeCGgcgv7/JdkINzi6Ay+dUvYFcAtwPzACGK+qx9x9/wvUBBa4V5934cwBn2oicK/7J99q4BcAVd0mIi8AvwK/AyuAFPc99wPvuO8phvMf170BOMdQy5rDBWekdoeqZojIp8D/icg8YBGwCkBVF7gXgBbhpNn8MRydDhZVVRG5ARgoIo/jfGcOAU8AXwPvi8hSIB24U1WPisi7uZS/Bwx1vyOLcP4BN0Fgt/aeIUQkXlUPuiPcccBHqjou3P0yxvzJphTOHM+6I8BlONMXX4W5P8aYU9gI1xhjQsRGuMYYEyIWcI0xJkQs4BpjTIhYwDWFIn9m71omImNFJC7/d+XaVhsRmeD+3klE+udRt4yI/LMQx8gxW1hu5afUGZbtLkAvx6opIssK2kdz5rOAaworK3tXI5zbSU9a2yuOAn+/VHW8qp66hjm7MkCBA64xkcACrgmEH4E67shupbvAfgFQXUTai5OndYE7Eo4HEJEOIrJKRGYBN2U1JCJ3yp+ZziqKyDhxcuAuFicH7ktAbXd0/Ypb7zH5M9Pac9naKlC2MBG5x21nsTgZ1rKP2q8SkR/Fych2nVs/xyxvxuTGAq4pEvdGi4646SNxAtsIVW2Gc+fTv4CrVLU5zn3+D4tIDPAhzu22l+Ik38nJIJykPE1wsn8tB/oD693R9WNu4p+6OFnVmgItROQyEcnKD9sMJ6Cf7+F0vlTV893jrcTJ4palJnA5cC3O3VoxZMvy5rZ/j4ic7eE45i/Kbu01hZX9VtsfgSFAFWCTqv7ill8INMBJNwlQApgN1Ac2qupaABH5BOjF6a4EbgdQ1QwgRUSSTqnT3t0Wuq/jcQJwAgXPFtZIRP6DM20Rz5+5LgDGqGomsFZENrjnkFuWtzUejmX+gizgmsI6kS4xixtUD2UvAqaoavdT6jUlcNmpBHhRVT845RgPFuIYw4AbVHWxiNwJtMm279S2lNyzvNUs4HHNX4RNKZhg+gW4RETqAIhInJulahVwtojUdut1z+X9U3GSY2fNlybiJFhPyFZnEnB3trnhqiJSgcJlC0sAtouTta3HKfu6iEiU2+daOMmDvGZ5MwawEa4JIlXd5Y4UR4mTdB3gX+5TMHoB34jIbpznvjXKoYkHgMEi0hPIAPq4uXB/cpddfefO454LzHZH2AeBWwuZLexpnIxrm3DmpLMH9tXADJzk5veq6hER8ZrlzRjAcikYY0zI2JSCMcaEiAVcY4wJEQu4xhgTIhZwjTEmRCzgGmNMiFjANcaYELGAa4wxIfL/zkDESwSZNNwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Best parameters set found on development set:\\n\", clf.best_params_)\n",
    "print(\"\\n Best Score\", clf.best_score_) \n",
    "\n",
    "print(\"\\n Grid scores on development set:\")\n",
    "means = clf.cv_results_['mean_test_score']\n",
    "stds = clf.cv_results_['std_test_score']\n",
    "for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "    print(\"%0.3f (+/-%0.03f) for %r\" % (mean, std, params))\n",
    "\n",
    "print(\"Detailed classification report:\")\n",
    "print(\"The scores are computed on the full evaluation set. \\n\")\n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(classification_report(y_true, y_pred))\n",
    "\n",
    "#Lets print confision matrix\n",
    "labels =['Average','Bad','Good']\n",
    "disp = plot_confusion_matrix(clf, X_test, y_test,\n",
    "                                 display_labels=labels,\n",
    "                                 cmap=plt.cm.Blues)\n",
    "\n",
    "print ('\\nTesting score: ',clf.score (X_test, y_test)) # print the testing score(accuracy)\n",
    "print ('\\nTraining Score: ',clf.score (X_train, y_train)) # print the trainig score(accuracy)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start time 21 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:    6.5s remaining:    9.8s\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   5 | elapsed:    6.6s remaining:    4.4s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 21.7 ms, sys: 46 µs, total: 21.7 ms\n",
      "Wall time: 6.77 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:    6.8s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:    6.8s finished\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "scoring = ['f1_macro']\n",
    "scores = cross_validate(clf, X, Y, cv=5, scoring=scoring, verbose=10, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "['fit_time', 'score_time', 'test_f1_macro']\n",
      "test_f1_macro [0.56048937 0.55362572 0.57829626 0.56671078 0.5703609 ]\n"
     ]
    }
   ],
   "source": [
    "print(\"-------------------------------------\")\n",
    "print(sorted(scores.keys()))\n",
    "print('test_f1_macro', scores['test_f1_macro'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.45579826472607415 0.7046019793738675 -0.28825772682504214\n",
      "  -0.10797909629417606]\n",
      " [-0.45821755567357747 -0.7059423617884181 -0.2916635351593642\n",
      "  -0.09416709513907746]\n",
      " [-0.21141491550725078 -0.0031579275616275076 0.8725130077194068\n",
      "  -0.3905247495057231]\n",
      " ...\n",
      " [-0.4581169085501422 0.717465384690377 -0.2917362036100829\n",
      "  -0.07432632961038373]\n",
      " [0.751538661872047 -0.0025887537747943763 -0.10000450010468652\n",
      "  -0.05167456137188623]\n",
      " [-0.4595648294413432 0.7112979984399956 -0.2918739685292939\n",
      "  -0.09370938251503302]] ['Bad' 'Bad' 'Bad' ... 'Average' 'Bad' 'Average']\n",
      "4800 4800 1200 1200\n"
     ]
    }
   ],
   "source": [
    "#Load Kbest data\n",
    "arr2= np.load('data_pca.npy', allow_pickle= True)\n",
    "X= arr2[:,0:-1]\n",
    "Y= arr2[:,-1]\n",
    "\n",
    "\n",
    "### Split arrays or matrices into random train and test subsets - 20% for Test, 80% to train\n",
    "X_train, X_test, y_train, y_test= train_test_split(X, Y, test_size=0.20,  random_state = 200)\n",
    "\n",
    "print(X_train,y_train)\n",
    "print(len(X_train),len(y_train),len(X_test),len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.47 s, sys: 6.18 ms, total: 6.47 s\n",
      "Wall time: 6.52 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score=nan,\n",
       "             estimator=LinearSVC(C=1.0, class_weight=None, dual=True,\n",
       "                                 fit_intercept=True, intercept_scaling=1,\n",
       "                                 loss='squared_hinge', max_iter=1000,\n",
       "                                 multi_class='ovr', penalty='l2',\n",
       "                                 random_state=None, tol=0.0001, verbose=0),\n",
       "             iid='deprecated', n_jobs=None,\n",
       "             param_grid=[{'C': [0.1, 1, 10, 100, 1000]}],\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1_macro', verbose=0)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "tuned_parameters = [{'C': [0.1,1, 10, 100, 1000]}]\n",
    "clf = GridSearchCV(LinearSVC(), tuned_parameters, scoring='f1_macro')\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters set found on development set:\n",
      " {'C': 100}\n",
      "\n",
      " Best Score 0.4694101175345947\n",
      "\n",
      " Grid scores on development set:\n",
      "0.469 (+/-0.020) for {'C': 0.1}\n",
      "0.469 (+/-0.020) for {'C': 1}\n",
      "0.469 (+/-0.020) for {'C': 10}\n",
      "0.469 (+/-0.020) for {'C': 100}\n",
      "0.353 (+/-0.037) for {'C': 1000}\n",
      "Detailed classification report:\n",
      "The scores are computed on the full evaluation set. \n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Average       0.46      0.28      0.35       405\n",
      "         Bad       0.53      0.49      0.51       392\n",
      "        Good       0.49      0.72      0.58       403\n",
      "\n",
      "    accuracy                           0.50      1200\n",
      "   macro avg       0.50      0.50      0.48      1200\n",
      "weighted avg       0.50      0.50      0.48      1200\n",
      "\n",
      "\n",
      "Testing score:  0.48276523868797655\n",
      "\n",
      "Training Score:  0.46904203445488185\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEGCAYAAAApAy29AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3wUVdfA8d9JQkJJIZCE3ouISLdgRUTF3hX0wYaiSHntYnvE3h5BBQSxIYqij4BieRSVroLSERGkSk8gERICpJ33j5mEACkbki2D5+tnP8neuTtz77ic3D17546oKsYYY/wvLNgNMMaYfwoLuMYYEyAWcI0xJkAs4BpjTIBYwDXGmACJCHYDQlX1GjW1dr2GwW5GyJJgN8ADdmRmBbsJIS09ZQv7dqeV660UHttINWevT3V1b8q3qtqjPMcrLwu4xahdryHvTJoW7GaErDCxkFuadxZtCnYTQtrkwdeWex+as5eoY67xqe6+xSMTyn3AcrKAa4zxMAHxTmbUAq4xxrsECAsPdit8ZgHXGONtHkpvWcA1xniYpRSMMSZwbIRrjDEBINgI1xhjAkNshGuMMQFjsxSMMSYQ7EszY4wJDMFSCsYYEzA2wjXGmECwlIIxxgSGAOH2pZkxxgSG5XCNMSYQLKVgjDGBYyNcY4wJEBvhGmNMAIhd2muMMYFjl/YaY0wg2JdmxhgTOJZSMMaYALD1cI0xJlAspWCMMYFjX5oZY0yAWA7XGGMCQCylYIwxgWMjXGOMCQyxgGuMMf7n3GHHAq4xxvifCBJmAfcf76XXJzN34Uqqx1Xj7ZcHHrb9+9lLmPD5bACqVI7krlsvplnjOuU6ZlZ2Di+MmMiqtVuIjanKY3ddQ+2keOYvXc1b478jJyeHiIgIbu99Hh3aNC3XscrrhdcnMXeBc37eHTrosO1/bU7hhZGT+HPdFvr0OodrLzmt3MfMys7hueGfFpyfx+++1jk/S1YzZvxUcnJyiYgI547e59Hx+GblPl55XdexHsfVjiF9fw7P/7D6sO2VI8K4oXN94qtWIkyEaX/uYN5ff5frmFUrhXPTiQ2oUbUSqZnZvPvLX+zNzqNz/TjObpkIQFZOHh8v3sKW3fvKdayK4qURrl+/3hORy0VERaSVP48Tis7r2oHnHr6h2O11kuIZNqQPb/1nAP+6sitDx0zxed/bktO4Z8jbh5X/b9oCoqtV4f3hd3PlhV14c/xUAOJiqvL0g9fz1ssDebD/FTw3/NOyd6iC9ejagRceubHY7THRVRh4y4Vcc3HZA+225DTuevytw8q/nraAmOgqjB9xD1dfdApvfPAtAHGxVXl28L94Z+hAHhpwZUicH4B5G9IY9eP6Yref3rQm29L388K0NQyfvY7Ljq9NuI/Bp3lCNa7vWO+w8u4tE1iVksHT3/3JqpQMznGD7M7MLF6bvZYXpq3mm5XJ9OxQ94j65A8i4tMjFPh7PkUvYA7Qs7w7EhHvzG4G2rZuTGx0lWK3H3dMQ2Lc7a1bNCBl566Cbd/NWsydD42m7/0jGTrmc3Lz8nw65k/z/+Dcru0BOPPk41j421pUlRZN6pJQIxaAxg2SyMrOISs750i7ViHatW5S4vmJj4umVfP6REQc/hb9btZi+g0exa33jeDlNz4jN9e38/Pjrys478wOQOifH4A1OzPJzM4toYYS5Z6fyIgwMrNyyVMFoFuLBO7t2pQHuzXn/FZJPh/z+Dqx/LLBGSX/suFvjq/jnJd1qXvZm+2c5/WpmVSvUukIeuQfFRVwRaSBiEwXkRUislxE/s8tHyIim0Vksfu4oNBrHhKR1SKyUkTOK+0Yfgu4IhINnAr0wQ24IvLxIY0dKyJXiki4iLwkIr+KyFIRud3d3tU9AR8Cy9yyz0RkgXtC+hbaVx8RWSUiM0TkTREZ4ZYnishEd9+/isip/urzkfrftAWc2KElABs2JTPjp9947anbGPNSf8LDwvhh9hKf9rMjdTdJNeMACA8Pp1rVKHanZx5UZ9a85bRoUofISt7MJm3YlMz0n5Yx/Om+vPWfAYSFhfH9nDKcn4QD5ye6qPMzdznNPXJ+Zq1NpXZMFE+dfwwPnd2ciUu3okCrpGgSq0Xy8oy1vDhtNQ3iK9OsZlWf9hkTFcHu/c4fm937c4iJOvw8dGkUz4rt6RXZlSMnZXiULge4V1WPBU4G+otIa3fbMFVt7z6+BnC39QSOA3oAr5c2MPTnu+oy4BtVXSUiqSLSEZgAXAt8LSKRwNlAP5ygvEtVTxCRKOBHEZnq7udEoI2qrnOf36KqqSJSBfhVRCYCUcBjQEcgHZgG5P8rfBXnZM0RkYbAt8Cxfux3mSz6bS3/m76AV568reD5n+u2cOdDowHYn5VN9dhqAPz7pQ/ZlpxGdk4uyTt20ff+kQBccUEXepzVEXVHNwcp9Jd9/cbtvDl+Ki8+cpN/O+VHC5etZdXaLdwxeBQAWVk5xMc55+exF8ezNTmNnJxctu/Yxa33jQDgygu7cP5ZnSjq9BQ+P+s2bmfM+G958dGb/N2NCnFsUjSbdu1j+Jz1JFSLpP+pjXlh2mqOSYqmVVI0D5zl5KGjIsJIjI5izc5M7jmzKRFhQlREGFUjwwvqTFm+nT+SM0o9ZouEapzcOJ5XZq0rtW4gCBWXLlDVrcBW9/d0EVkBHJ53OeBSYIKq7gfWichqnHj1c3Ev8GfA7QW84v4+wX3+GPCaG1R7ALNUda+InAu0FZGr3PpxQAsgC/ilULAFGCQil7u/N3Dr1QZmqmoqgIj8F2jp1ukOtC70PyVWRGJU9bA/0e6IuS9Arbr1y9V5X6zZsI2X3/iM5x66gbgYZwSiCuee2Z5brzv3sPpP3n8d4OQoX3x9EkOH9Dloe2LNOJJ37iKxZhy5ubnsydxf8LE9Zecu/v2fjxjc/0rq1q7h5575j6Kcd2YHbrv+8PPz1APXA875eX7kRF554taDtifWjCV5x4Hzk3Ho+XnpQwYPuIp6tWv6vyMV4KRG8Xy3KgWAHXuy2JmZRVJMFAJ8tyqFn9anHfaaoTPXAk4O96SG1Rm/cPNB29P35xDrjnJjoyJI338gtVI3NopeHeox6uf1ZGaVlOoIrLCwiv+gLiKNgQ7APJxP6gNE5AZgPs4oOA0nGM8t9LJNlByg/ZNSEJGaQDfgLRFZD9yPM7LdD8wAznOfT8h/CTCw0JC9iarmj3D3FNpvV5wA2kVV2wGLgMqU/IEhzK2fv+96RQVbAFUdo6qdVbVz9RoJR9J1n23f8TdD/vMRDw24igZ1Dxyrw/FNmTV3OWm7nNHG7oxMtqf49s1zl06tmDpjMQAz5y6nw3FNEBEy9uzl4eff59Ze59CmVaOK70wAdWzTjJmFz096JttSDg8sRTmlcyu+nbkIcM9Pm6YF52fwc+9z63XncryHzk9aZhbHJEYDEBMVTlJ0FDv3ZPFHcgYnN4onMtz55x1XOYLoSN++Avlt225ObFQdgBMbVWfZ1t0AxFepRJ+TGvL+go2kZGT5oTdHrgw53AQRmV/o0beY/UUDE4G7VHU3MApoBrTHGQG/nF+1iJcX9TmqgL9GuFcB41T19vwCEZkJnIYTZG8FOgM3uZu/BfqJyDRVzRaRlsDBf3odcUCaqma6Mx9Odst/AYaJSDxOSuFK3JwvMBUYALzktqO9qi6usJ4W4+lXPmHJ7+vYlZ7JtXe8xI3XdCM3xxkVXHzuibz/6Qx2Z2Ty6ltfABAeHsao5/vRuH4SN/fszoNPv0eeKhHh4QzqcxG1EquXeswLunXkuRET6T1wGDHRVXj0rmsA+OybeWzZlsoHE2fwwcQZALzw6I3Ex0X7p/M+eOqVj1m83Dk/V9/+Ijdd063gy69Lzj2R1LR0bh88isy9+xERPv3qJ8YOG0TjBknc0rM79z81FlUlPDycu269mNqJ8aUe88JunXh2+KdcP2AosdFVeOzuawGY/M1ctmzbyfufTuf9T6cD8NJjNwX1/ADc2Lk+zROrER0ZwZM9juHrFcmEu//Ef1yfxjcrU/hXx/oM7tYcBKYs38aerFz+SM6gVkwU95zpTP3bn5vH+/M3keHDqPS7VTu4+YQGnNwonrTMbN79ZSMAPVolUi0ygqvbObMT8hT+M2ONfzpeFr7nZwF2qGrnEncnUgkn2I5X1UkAqrq90PY3gS/dp5twPmXnqw9sKXH/Reb9yklEZgDPq+o3hcoG4eROBwHbgCmqerO7LQx4GrgY5/Sl4OSAOwD3qepFbr0o4DOcYftKIBEYoqoz3L9W9+F0eAWQqqqPiEgCMNI9dgROGuOO0vrQ6vgO+s6kaeU9FUetsBCZZhPK3lm0KdhNCGmTB19Lyprl5XojRSQ01eoXPetT3Z3v9VpQUsAVZxj8Hk7suKtQeR03v4uI3A2cpKo9ReQ44EOcvG1d4AeghaoW+5fNLyNcVe1aRNlrhZ7WPGRbHvCw+yhshvvIr7cfOL+Yw36oqmNEJAKYjDOyRVV34KQvjDFHmYr80gwnV9sbWCYi+Z+CHwZ6iUh7nHTBeuB2AFVdLiKfAL/jzHDoX1KwhaPrSrMhItIdJ6c7FWckbIw5ylXUpb2qOoeiExRfl/CaZ4BnfD3GURNwVfW+YLfBGBNg4q1Le4+agGuM+WeygGuMMQFiAdcYYwKggr808zsLuMYYb/NOvLWAa4zxMPHPpb3+YgHXGONpllIwxphA8U68tYBrjPE2G+EaY0wA+Ho3h1BhAdcY42kWcI0xJkDsNunGGBMgNsI1xphAsMVrjDEmMISD7gMa8izgGmM8zGYpGGNMwITZl2bGGBMAYikFY4wJCMFGuMYYEzA2wjXGmACxL82MMSYQLIdrjDGBIYgtQG6MMYFiI1xjjAkQy+EaY0wgWA7XGGMCw1lLwTsR1wKuMcbTPBRvLeAaY7zNrjQzxphAsPVwjw5REWE0TqwW7GaErDs/XRrsJoS8Aac2DnYTQtqMKpXKvQ9bD9cYYwLG1sM1xpiA8VC8tYBrjPEwsS/NjDEmIGwerjHGBJAFXGOMCRAPxVsLuMYYb/PSCNc7C0kaY8yh3MVrfHmUuiuRBiIyXURWiMhyEfk/t7yGiHwnIn+6P+PdchGR10RktYgsFZGOpR3DAq4xxrOcBch9e/ggB7hXVY8FTgb6i0hrYDDwg6q2AH5wnwOcD7RwH32BUaUdwAKuMcbTwkR8epRGVbeq6kL393RgBVAPuBR4z632HnCZ+/ulwDh1zAWqi0idEtt6ZF00xpjQUIaUQoKIzC/06Fv8PqUx0AGYB9RS1a3gBGUgya1WD9hY6GWb3LJi2ZdmxhjPkrItXrNDVTuXvk+JBiYCd6nq7hL2X9QGLWnfxQZcEYkt6YWquruk7cYYEwgVeaGZiFTCCbbjVXWSW7xdROqo6lY3ZZDslm8CGhR6eX1gS0n7L2mEuxwnWhfuTv5zBRr63AtjjPGTirq0V5yh7NvAClUdWmjTFOBG4Hn35+eFygeIyATgJGBXfuqhOMUGXFVtUNw2Y4wJBYIzU6GCnAr0BpaJyGK37GGcQPuJiPQB/gKudrd9DVwArAYygZtLO4BPOVwR6Qk0VdVnRaQ+ThJ5QVl6Yowx/lBRKQVVnUPReVmAs4uor0D/shyj1FkKIjICOAsn8oMTyUeX5SDGGOMX4qyH68sjFPgywj1FVTuKyCIAVU0VkUg/t8sYY3wSIrHUJ74E3GwRCcOd7iAiNYE8v7bKGGN8IODTRQ2hwpeAOxJnmkSiiDwBXAM84ddWGWOMj46qBchVdZyILAC6u0VXq+pv/m2WMcaUzteFaUKFr1eahQPZOGkFuxzYGBMyvJRS8GWWwiPAR0BdnCspPhSRh/zdMGOM8YX4+AgFvoxw/wV0UtVMABF5BlgAPOfPhhljjC9CZcqXL3wJuBsOqRcBrPVPc4wxxnfOLIVgt8J3JS1eMwwnZ5sJLBeRb93n5wJzAtM8Y4wpgfi8uHhIKGmEmz8TYTnwVaHyuf5rjjHGlM1RkVJQ1bcD2RBjjCmroyalkE9EmgHPAK2ByvnlqtrSj+0yxhifeGmE68uc2rHAuzh/TM4HPgEm+LFNxhjjMy9NC/Ml4FZV1W8BVHWNqj6Ks3qYMcYElQiEh4lPj1Dgy7Sw/e5K6GtE5A5gMwduomb8YO1fyQx6clzB841bd3LXzT1YuHwD6zY6d/fYnbGX2OgqfPnWfcFqZrncfmpjOtaPY/e+HO7/fPlh26tFhnP7qY2pFRNFdq4y+sd1bPp7X7mOGREm9D+9CU1qViVjfw6vzlxLSkYWx9eJpVenekSECzm5yvj5m1i+Lb1cx6oIr77xGb8uWkVcbDVGvnj4sqsz5ixl4hfOhKHKlSO585aLaNKodrmOmZ2dw9BRk1mzbgsx0VV5YNBV1EqMZ9GyNbz30ffk5OYSER7OzdefQ7vjmpbrWBXFSykFXwLu3UA0MAgnlxsH3OLPRpVERHKBZTifEnKBAar6UxlePwTIUNX/+KeF5de0YVJBIM3NzeOUq5/g3NOO5+arziyo8+zrnxNTrXJxuwh5M1fv4NsVyfQ/vUmR2y9rW4cNqZkMnb6GunGVueWkhjw9dZVP+06MjqTfaU148puVB5Wf1SKBjKwc7pr0G12axHNdp/q8OnMt6fuzeemH1aTtzaZ+9co8fE5L7vzv0nL3sbzOPqM9F557IsNGTS5ye62k6jz32M1ER1dh/uI/GfHWF7z81G0+7Xt7ShqvjP6M5x47+CYFU2csJLpaZcYM+z9m/bSMsR99z4ODriY2piqP3d+LmvGxbNi4nX8//wHvjby33H2sCB6Ktz4tXjPP/TWdA4uQB9NeVW0PICLn4VzxdmbJL/Gunxb+ScO6NalXu0ZBmary1YwlfDC0XxBbVj5/bM8gMbr4ZZXrxVXm82XbANiyax+J0ZHEVY5g174cTmtagx7H1iIiXFidsoe3525AS7xXqqNzw+p8uti5x9+89WncfJJzW771qXsL6mz6ex+VwsOICBNy8nzYqR+1ObYx21PSit1+bMsDtxVs1bw+O1IP3Nd1+pwlfPHNPHJyc2nZrD79brmQ8LDSM4jz5q/kuiu7AnDqSa0ZPfZrVJVmjesU1GlYP4ns7Byys3OoVCm4N/4WxFNrKZR04cNkSrjlr6pe4ZcWlU0skAYFtzb+HIgHKgGPqurn7rZHgBtw7iGfgnNpsid8OW0RF5/d4aCyX5euJSE+mib1E4PUKv/7K20vJzaqzsrkDJolVCMhOooa1SKpFhVBlyY1ePzrP8hV5ZaTG3Ja05rMXrOz1H3WqBrJzj1ZAOQp7M3KJSYqgvT9OQV1TmoUz/rUzKAH27KaOmMhndo1B2Dj5hRm/7ycF4f0ISIinNff+ZKZc5bS7Yz2pe5nZ9puEmo6N+wODw+nWtXK7E7PJC62WkGdn375naaNagc92AJwFK0WNiJgrSibKu4N3ioDdYBubvk+4HL3PvIJwFwRmQJ0BHoCHXD6u5BiAq6I9AX6AtSrH/ybEmdl5/DDT8u5/7YLDyr/YtoiLj67Y5BaFRifL9vKjSc25PlLWrMxbS/rUzPJzVOOrxNDk5pVeebiYwGIDA9j9z4nYN5zVjOSYqKICBMSqkXy/CWtAfjf79uZubrogKyFxhT1q1fmuk71ePa7P/3cu4q1dPk6vpuxiBcedzJ9S35by5p1W7jnsTEAZGXlUN0NmM8MncD2lDRycnJJ2bGLQQ+NAuCS806me9cORX5SKJwj3bApmbEffc+TD4XCh13HUZHDVdUfAtmQMiicUugCjBORNjg53WdF5AycO1LUA2oBpwOTCy2+M6W4HavqGGAMQLsOnYI+xJk57w+Oa1mPhBoxBWU5ubl8O3spn79xTxBb5n97s/MY/eP6gufDrzqelIz9HFsrhlmrdzJh4ebDXjN0+hqg+BxuamYWNatFkpqZTZhAlchwMvbnAlCjaiXuPas5I+esZ3v6fv91rIKt+2sbw9+cwpAHryc2pirgfCztdkZ7buzZ/bD6j9zTEyg+h5tQI5YdO3eTUDOO3Nxc9mTuIya6CgA7du7i2aETuLvf5dSpVeOwfQeDAOEeCrieXttWVX8GEoBE4Hr3Zyc3IG/nwIUaQQ+eR+KLaQu5uNvBI9kfF6yiWYMk6iRWD1KrAqNqZHjBVJ5uLRJYsS2dvdl5/LZ1Nyc1jie2sjNWqBYZTkI1326xt2Dj35zRvCYAJzWOZ/nW9IJjPdi9BR8t3MSq5Aw/9MY/knf8zXPDPuaeOy+nXp2EgvJ2xzXhx3m/8/cupy/pGZkkp/zt0z5P6nQMP8x27hD+47zfaXtcE0SEjD17eeKlD7mhZ3daHxP8T3+FhYlvj1AQAkmYIycirXAWR9+JM3siWVWzReQsoJFbbRYwVkSex+nvxcAbwWhvWezdl8WPC1bxzD1XH1T+5bTFR0U6YeAZTWhdO4aYyhGMvLotny7eUhBgv1+ZQr24ytx5ehPyVNn89z7ecEe7m3ft45OFm3n43JbONJU85Z15f7HDzc2WZPqfO+h/ehNeuaINGftzeW2mMyI+r1UStWKiuKJdXa5oVxeAZ6euKkhVBMtLwz9l2Yr17E7P5KYBL3PdlWeRm+uMyM/vfgITJs1kd/peRr3rLHUSHhbGsGdup2H9JHpf041/P/8+mqeEh4dzx80XkOTDH+lzunZg6OuT6Xv3q0RXq8IDA68C4Kupv7B1eyofT57Jx5NnAvDk4N5Uj4v2U+99FyrB1Beivny9C4hIlKoG/bNWoWlh4HyieFhVv3Lztl/gfGG2GDgVOF9V1xf60mwDsAn4vbRpYe06dNL/Tf/ZX93wvDs/Df60qVA34NTGwW5CSLvz6u6s+m1xucJl7RZt9PqhE32qO/SSVgtUtXN5jldevqylcCLwNs4IsqGItANuVdWB/m5cUVQ1vJjyHUCXYrY9gzOH2BhzlPHSCNeXHO5rwEU4H9tR1SXYpb3GmBCRfyPJ0h6hwJccbpiqbjhk6kWun9pjjDE+EyAiVKKpD3wJuBvdtIKKSDgwEPDtGktjjPEzD8VbnwJuP5y0QkOcqVbfu2XGGBNUIkfJpb35VDUZ50otY4wJOR6Ktz7NUniTIi4cUNW+fmmRMcaUgZdmKfiSUvi+0O+VgctxFoExxpigEgiZxcV94UtK4ePCz0XkfeA7v7XIGGN8FUKX7friSC7tbcKBy2aNMSaoJGTuWFY6X3K4aRzI4YYBqcBgfzbKGGN8cVTdJt29l1k7nPuYAeSpr4svGGNMAHgp4JZ4aa8bXCeraq77sGBrjAkpIuLTIxT4spbCLyLi/fUAjTFHHec26b49St+XvCMiySLyW6GyISKyWUQWu48LCm17SERWi8hK9/6KpSrpnmYRqpoDnAbcJiJrgD04aRNVVQvCxpigq8Arzcbi3Fps3CHlww5dzlVEWuNcEHYcUBf4XkRaqmqJ68yUlMP9Bed+YJeVsdHGGBMQFfmlmarOEpHGPla/FJjgrhG+TkRWAycCJS6iXVLAFbcRa3xsgDHGBFwZBrgJIjK/0PMx7n0MSzNARG4A5gP3qmoazj0T5xaqs8ktK1FJATdRRIq9U6GqDvWhocYY40dCmO/zcHccwR0fRgFP4UyNfQp4GbgFijxoqZMKSgq44UB0MTs2xpigE/y7eI2qbi84lrOuzJfu001Ag0JV6wNbSttfSQF3q6o+eSSNNMaYgBCI8ONEXBGpo6pb3aeXA/kzGKYAH4rIUJwvzVrgfO9VolJzuMYYE6oqcoQrIh8BXXFyvZuAx4GuItIeJ12wHrgdQFWXi8gnwO9ADtC/tBkKUHLAPbtcrTfGmACoqGlhqtqriOK3S6hf5pvTFhtwVTW1LDsyxphgCJGLyHxyJKuFGWNMSBB8u1w2VFjANcZ4l1TolWZ+ZwHXGONZzpVmFnCNMSYgvBNuLeAaYzzOQwNcC7jGGC8LnbVufWEB1xjjWTZLwRhjAsi+NDsKhIcJ8dUqBbsZIWvCTWVddOmfJ/6EAcFuQkjbv7bUtV5KJ1hKwRhjAsFSCsYYE0A2wjXGmADxTri1gGuM8TABwm2Ea4wxgeGheGsB1xjjZYJ4KKlgAdcY42k2wjXGmABwpoV5J+JawDXGeJfYCNcYYwLGLu01xpgAcBYgD3YrfGcB1xjjaTZLwRhjAsRDGQULuMYYb7MRrjHGBIDlcI0xJlBEbJaCMcYEinfCrQVcY4yHOSkF74RcC7jGGE/zTri1gGuM8ToPRVwLuMYYT7OUgjHGBIh3wq0FXGOM13ko4lrANcZ4lmBXmhljTGDYerjGGBM4Hoq3FnCNMV4miIeGuBZwjTGe5qF4S1iwG2CMMUdKyvAodV8i74hIsoj8Vqishoh8JyJ/uj/j3XIRkddEZLWILBWRjr601wKuMcbbKiriwligxyFlg4EfVLUF8IP7HOB8oIX76AuM8uUAFnCNMZ4mPv5XGlWdBaQeUnwp8J77+3vAZYXKx6ljLlBdROqUdgzL4Yaodpc+TnTVKMLDwogID2PauAe45eF3WL0hGYBdGXuJi67CrPGDS9nT0antJf8+cH4iwpg+7kEAxnw8gzc/mUVEeBjnnNaGJwddVsqeQlO9WtUZNeQGkmrGkqfKe5N/5I0JMw6qExdThRGP/Ysm9RPYl5XNwKfGs2LN1nIdN7JSBKOe6E37Vg1J3bWHWx5+h41bU+l6YiseH3AJkZUiyMrO4d+vfcbs+avKdayKUoYcboKIzC/0fIyqjinlNbVUdSuAqm4VkSS3vB6wsVC9TW5Zif8DQjbgikgtYBhwMpAGZAEvqurkcu53BnCfqs4vrW6wTRk1iJrVowuev/PsLQW/P/rKJGKjqwSjWSHji9H/d9D5mT1/FV/PXMacjx4iKrISKanpQWxd+eTk5PHoK5NYunIT0VWjmD7uQWbM+4OV67YV1Ln35vNYtmoTvR94kxaNavHSg9dw2Z3Dfdp/gzo1eP3x3lx8x6sHlfe+tAu7du+l0xVPcMU5nRgy8FL6PPwuO//OoNc9b7Btxy6ObVaHT1/rz3EXPlqhfT4iZZuHu0NVO1fckQ+jpb0oJFMK4szz+AyYpapNVbUT0BOoH9yWhQZV5bPvF3HluZ2C3ZSQ8gbVGsIAAA4RSURBVM7E2dx14zlERVYCILFGTJBbdOS279zN0pWbAMjI3M+q9duok1j9oDrHNKnNrF9XAvDnhu00rFOjoM/XnH8C34+9j1njBzPsoZ6E+XgfmvPPaMtHX80D4PNpizjzhGMAWLZqE9t27AJgxZqtVI6sRGSl0BivVVRKoRjb81MF7s9kt3wT0KBQvfrAltJ2FpIBF+gGZKnq6PwCVd2gqsNFpLKIvCsiy0RkkYicBVBCeRURmeB+k/gx4IlhoQBXDhzJWTe8yNjJPx607edFa0iqEUOzhklFv/gfQES4YsAIuvZ+gbGT5gCwekMyPy9eQ/ebXuLCvq+wcPmGILeyYjSoU4O2x9RnwfL1B5X/9udmLjqrPQAdWzeiQe0a1E2qTsvGtbj8nI706DOUM65/nty8PK7ucYJPx6qbFMfm7WkA5ObmsTtjLzXiqh1U55Ju7Vm6aiNZ2Tnl71w5Cc4I15fHEZoC3Oj+fiPweaHyG9zZCicDu/JTDyUJjT9RhzsOWFjMtv4Aqnq8iLQCpopIyxLK+wGZqtpWRNqWsN+Q8r+37qFOYhwpqelcMWAELRvV4pSOzQGYOHUBV5z3zx7dfvPW3dRJrE5KajqXDxhBi8a1ycnN4+/0TL579z4W/r6Bmx9+h8WfDfHUxPhDVasSybgXbuWhoRNJ37PvoG2vvPcdz917FbPGD+b31VtYumoTubl5nHnCMbRr1ZBp4x4AoHJUJVJSMwB4/8XbaFSvJpUiwqlfu0bBdwCjJ8zgwy/mFhmZCn9ObtW0NkMGXsoVA0b6p8NHoKL+74rIR0BXnFzvJuBx4HngExHpA/wFXO1W/xq4AFgNZAI3+3KMUA24BxGRkcBpOHncTcBwAFX9Q0Q2AC3d7UWVnwG85pYvFZGlJRynL84UDxo0bOi3/viiTmIc4HwsvrBrOxb8voFTOjYnJyeXL2csYdp79we1fcGW//E6sUYMF3Vty8Ll66mXVJ2Lz2qHiNDpuMaEibDz7wwS4r2ZWogID+O9F27jv9/M58vpSw7bnr5nHwOe/KDg+ZLPn2DDlp106dCcCV/N48mRUw57Te8H3gSKz+Fu2f439WrFsyX5b8LDw4iNrkLarj0A1E2qzvsv9qXf4++zfvOOiuxq+VRQxFXVXsVsOruIuoo7yCuLUE0pLAcKJhKran+cTidS/Okt6bSXmsx2jzNGVTuraueEhERf21rh9uzdXzCa2bN3P9Pn/cGxzZwZJzN+XUmLRrWoVys+aO0LtkPPz7S5f3Bss7pc0LUts351vjlfvWE7Wdk5B32p5jXDH7ueVeu38fqH04rcHhtdhUoR4QDccNkp/LRoNel79jHr15Vc0q09CfFO36vHVqVBbd/eL9/MXkavC08C4NJuHQrOZ2x0FT4edgdPjpzCvKVry9u1ChXm3rm3tEcoCNUR7jTgWRHpp6r5E4qruj9nAdcD09yUQUNgpQ/l00WkDdA2cN04Mimp6fS+3xmJ5OTmcdV5nenepTUAk6cu+Md/WZayM51/uSO13JxcruzRme6ntCYrO4cBT46ny7XPEFkpnFFDens2nXByu6b0vPAklv+5ueBj/1Mjp1C/dg0A3p00h2Oa1GbUkN7k5uWxct02Bj41HoCV67bxzOgvmTRiAGEiZOfkcv+Ln7BxW1qpx33/858Y/cQNLJj0OGm799DnkXcBuO2aM2jSIJH7b+3B/bc61wZcMWAEO9Iy/NH9MvHS/2FxRsahx/1GcBhwEpAC7AFG4yStRwOdgBzgHlWdLiKViymvArwLtAYWA82BQaVNC+vYqbP+OPdXv/TtaODVQBZI8ScMCHYTQtr+lZ+Ql5lcrjdSm3YdddLUOT7VPaZ2tQUVOC3siITqCBf3G7+exWy+qYj6+4op31vCfowxHmYLkBtjTKDYAuTGGBM4Hoq3FnCNMV5mC5AbY0zAeCjeWsA1xniX70vdhgYLuMYYb/NQxLWAa4zxNJsWZowxAWI5XGOMCQQBH5f6DQkWcI0xHuediGsB1xjjWfkLkHuFBVxjjKd5KN5awDXGeJuNcI0xJkDs0l5jjAkQ74RbC7jGGA8r5x15A84CrjHG0+xKM2OMCRTvxFsLuMYYb/NQvLWAa4zxstC5BbovLOAaYzzLa1eahQW7AcYY809hI1xjjKd5aYRrAdcY42k2LcwYYwLBLnwwxpjA8NqXZhZwjTGeZikFY4wJEBvhGmNMgHgo3lrANcZ4nIcirgVcY4xnCXjq0l5R1WC3ISSJSAqwIdjtKCQB2BHsRoQ4O0clC7Xz00hVE8uzAxH5Bqdfvtihqj3Kc7zysoDrESIyX1U7B7sdoczOUcns/ASfraVgjDEBYgHXGGMCxAKud4wJdgM8wM5Ryez8BJnlcI0xJkBshGuMMQFiAdcYYwLEAq4ficjlIqIi0irYbfE6EckVkcUiskREForIKWV8/RARuc9f7QsGEaklIh+KyFoRWSAiP4vI5RWw3xkiYtPH/MACrn/1AuYAPcu7IxEJL39zPG2vqrZX1XbAQ8BzwW5QMImIAJ8Bs1S1qap2wnmf1Q9uy0xJLOD6iYhEA6cCfXADroh8LCIXFKozVkSuFJFwEXlJRH4VkaUicru7vauITBeRD4Flbtln7mhmuYj0LbSvPiKyyh2dvCkiI9zyRBGZ6O77VxE5NXBnwW9igTRwzrOI/OCOepeJyKX5lUTkERFZKSLfA8cEq7F+0g3IUtXR+QWqukFVh4tIZRF51z0fi0TkLIASyquIyAT3vfcxUCU4XTr62VoK/nMZ8I2qrhKRVBHpCEwArgW+FpFI4GygH05Q3qWqJ4hIFPCjiEx193Mi0EZV17nPb1HVVBGpAvwqIhOBKOAxoCOQDkwDlrj1XwWGqeocEWkIfAsc6+e++0MVEVkMVAbq4AQcgH3A5aq6W0QSgLkiMgXnXPQEOuC8zxcCCwLfbL85DqdPRekPoKrHu+msqSLSsoTyfkCmqrYVkbYl7NeUkwVc/+kFvOL+PsF9/hjwmhtUe+B8HNwrIucCbUXkKrd+HNACyAJ+KRRsAQYVytM1cOvVBmaqaiqAiPwXaOnW6Q60lgMLfMSKSIyqpldsd/1ur6q2BxCRLsA4EWmDs37JsyJyBpAH1ANqAacDk1U1033NlOA0OzBEZCRwGs57ZhMwHEBV/xCRDTjvh9OKKT8DeM0tXyoiSwPfg38GC7h+ICI1cUZgbUREgXBAgQeAGcB5OCPdj/JfAgxU1W8P2U9XYM8hz7sDXVQ1U0Rm4Iz4SlouKcytv7e8/QoVqvqzO5pNBC5wf3ZS1WwRWY9zTsA550er5cCV+U9Utb97TuYDm4t5TUnvk6P5XIUMy+H6x1XAOFVtpKqNVbUBsA5nhDEBuBlnBJYfYL8F+olIJQARaSki1YrYbxyQ5gbbVsDJbvkvwJkiEi8iERT6hwhMBQbkPxGR9hXWyyBx+x4O7MQ5J8lusD0LaORWmwVc7uYnY4CLg9Nav5kGVBaRfoXKqro/ZwHXg/NeAhoCK30sbwO0DUD7/5FshOsfvYDnDymbCFwHDALGAVNUNcvd9hbQGFjofvucgpMDPtQ3wB3uR76VwFwAVd0sIs8C84AtwO/ALvc1g4CR7msicP5x3VEBfQy0/BwuOCO1G1U1V0TGA1+IyHxgMfAHgKoudL8AWoyzzObsYDTaX1RVReQyYJiIPIDzntkDPAh8DowWkWVADnCTqu4XkdeLKR8FvOu+Rxbj/AE3fmCX9h4lRCRaVTPcEe5k4B1VnRzsdhljDrCUwtFjiDsC/A0nffFZkNtjjDmEjXCNMSZAbIRrjDEBYgHXGGMCxAKuMcYEiAVcc0TkwOpdv4nIf0WkaumvKnZfXUXkS/f3S0RkcAl1q4vInUdwjCJXCyuu/JA6YwtdBejLsRqLyG9lbaM5+lnANUcqf/WuNjiXkx40t1ccZX5/qeoUVT10DnNh1YEyB1xjQoEFXFMRZgPN3ZHdCneC/UKggYicK846rQvdkXA0gIj0EJE/RGQOcEX+jkTkJjmw0lktEZkszhq4S8RZA/d5oJk7un7JrXe/HFhp7YlC+yrTamEicpu7nyXirLBWeNTeXURmi7Mi20Vu/SJXeTOmOBZwTbm4F1qcj7t8JE5gG6eqHXCufHoU6K6qHXGu879HRCoDb+Jcbns6zuI7RXkNZ1Gedjirfy0HBgNr3NH1/e7CPy1wVlVrD3QSkTNEJH992A44Af0EH7ozSVVPcI+3AmcVt3yNgTOBC3Gu1qpMoVXe3P3fJiJNfDiO+YeyS3vNkSp8qe1s4G2gLrBBVee65ScDrXGWmwSIBH4GWgHrVPVPABH5AOjL4boBNwCoai6wS0TiD6lzrvtY5D6PxgnAMZR9tbA2IvI0TtoimgNrXQB8oqp5wJ8istbtQ3GrvK3y4VjmH8gCrjlSBcsl5nOD6p7CRcB3qtrrkHrtqbjVqQR4TlXfOOQYdx3BMcYCl6nqEhG5CehaaNuh+1KKX+WtcRmPa/4hLKVg/GkucKqINAcQkaruKlV/AE1EpJlbr1cxr/8BZ3Hs/HxpLM4C6zGF6nwL3FIoN1xPRJI4stXCYoCt4qzadv0h264WkTC3zU1xFg/ydZU3YwAb4Ro/UtUUd6T4kTiLrgM86t4Foy/wlYjswLnvW5sidvF/wBgR6QPkAv3ctXB/dKdd/c/N4x4L/OyOsDOAfx3hamGP4ay4tgEnJ104sK8EZuIsbn6Hqu4TEV9XeTMGsLUUjDEmYCylYIwxAWIB1xhjAsQCrjHGBIgFXGOMCRALuMYYEyAWcI0xJkAs4BpjTID8PwL+I1tFToKfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Best parameters set found on development set:\\n\", clf.best_params_)\n",
    "print(\"\\n Best Score\", clf.best_score_) \n",
    "\n",
    "print(\"\\n Grid scores on development set:\")\n",
    "means = clf.cv_results_['mean_test_score']\n",
    "stds = clf.cv_results_['std_test_score']\n",
    "for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "    print(\"%0.3f (+/-%0.03f) for %r\" % (mean, std, params))\n",
    "\n",
    "print(\"Detailed classification report:\")\n",
    "print(\"The scores are computed on the full evaluation set. \\n\")\n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(classification_report(y_true, y_pred))\n",
    "\n",
    "#Lets print confision matrix\n",
    "labels =['Average','Bad','Good']\n",
    "disp = plot_confusion_matrix(clf, X_test, y_test,\n",
    "                                 display_labels=labels,\n",
    "                                 cmap=plt.cm.Blues)\n",
    "\n",
    "print ('\\nTesting score: ',clf.score (X_test, y_test)) # print the testing score(accuracy)\n",
    "print ('\\nTraining Score: ',clf.score (X_train, y_train)) # print the trainig score(accuracy)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start time 21 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:    6.7s remaining:   10.0s\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   5 | elapsed:    6.8s remaining:    4.5s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 19 ms, sys: 69 µs, total: 19 ms\n",
      "Wall time: 7.36 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:    7.4s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:    7.4s finished\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "scoring = ['f1_macro']\n",
    "scores = cross_validate(clf, X, Y, cv=5, scoring=scoring, verbose=10, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "['fit_time', 'score_time', 'test_f1_macro']\n",
      "test_f1_macro [0.47942543 0.46803293 0.47266061 0.46363159 0.47202167]\n"
     ]
    }
   ],
   "source": [
    "print(\"-------------------------------------\")\n",
    "print(sorted(scores.keys()))\n",
    "print('test_f1_macro', scores['test_f1_macro'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}