GridSearch #4 Naive Bayes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pylab as pl\n",
    "from sklearn.model_selection import train_test_split\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.metrics import plot_confusion_matrix\n",
    "\n",
    "from sklearn import datasets\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.000210523684764536 0.005160091115494809 0.1361031518624642 ... 0 0 0]\n",
      " [0.06578947122576177 0.013897588526412541 0.14469914040114612 ... 0 0 0]\n",
      " [0.005263155277008304 0.010694145453587434 0.11891117478510028 ... 0 1 0]\n",
      " ...\n",
      " [0.05263157645429362 0.859810398981424 0.1346704871060172 ... 0 0 0]\n",
      " [0.000988368423653601 0.005792679656356735 0.1017191977077364 ... 1 0 0]\n",
      " [0.17105262939750693 0.057828596847734885 0.13753581661891118 ... 0 0 0]] ['Bad' 'Bad' 'Bad' ... 'Average' 'Bad' 'Average']\n",
      "4800 4800 1200 1200\n"
     ]
    }
   ],
   "source": [
    "arr2= np.load('data_array_formatted.npy', allow_pickle= True)\n",
    "X= arr2[:,0:-1]\n",
    "Y= arr2[:,-1]\n",
    "\n",
    "\n",
    "### Split arrays or matrices into random train and test subsets - 20% for Test, 80% to train\n",
    "X_train, X_test, y_train, y_test= train_test_split(X, Y, test_size=0.20,  random_state = 200)\n",
    "\n",
    "print(X_train,y_train)\n",
    "print(len(X_train),len(y_train),len(X_test),len(y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters set found on development set:\n",
      " {'var_smoothing': 1e-06}\n",
      "\n",
      " Best Score 0.5138273887222787\n",
      "\n",
      " Grid scores on development set:\n",
      "0.514 (+/-0.013) for {'var_smoothing': 1e-06}\n",
      "0.514 (+/-0.013) for {'var_smoothing': 1e-07}\n",
      "0.514 (+/-0.013) for {'var_smoothing': 1e-08}\n",
      "0.514 (+/-0.013) for {'var_smoothing': 1e-09}\n",
      "0.514 (+/-0.013) for {'var_smoothing': 1e-10}\n",
      "0.514 (+/-0.013) for {'var_smoothing': 1e-11}\n",
      "0.514 (+/-0.013) for {'var_smoothing': 1e-12}\n",
      "0.514 (+/-0.013) for {'var_smoothing': 1e-13}\n",
      "0.514 (+/-0.013) for {'var_smoothing': 1e-14}\n",
      "0.514 (+/-0.013) for {'var_smoothing': 1e-15}\n",
      "Detailed classification report:\n",
      "The scores are computed on the full evaluation set. \n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Average       0.54      0.23      0.33       405\n",
      "         Bad       0.52      0.83      0.64       392\n",
      "        Good       0.59      0.60      0.60       403\n",
      "\n",
      "    accuracy                           0.55      1200\n",
      "   macro avg       0.55      0.55      0.52      1200\n",
      "weighted avg       0.55      0.55      0.52      1200\n",
      "\n",
      "\n",
      "Testing score:  0.5214443474620064\n",
      "\n",
      "Training Score:  0.5205784550832218\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEKCAYAAABewe3GAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd8FVXawPHfkwSSkEYk9CJFqkhXQNCl6YqLgrq6gAXLCmJdV13BtuiuymtZFAuKDdBFdFUEsYEoTelIL9K7QCCEQEJJeN4/ZoIBkpubcsvg8/Uzn9x75tw5Z643DyfnzjxHVBVjjDGBFxHqDhhjzO+FBVxjjAkSC7jGGBMkFnCNMSZILOAaY0yQWMA1xpggsYBrjDFBYgHXGGOCxAKuMcYESVSoOxCuyp9VQatVrxXqboStPZlHQ92FsJcYY79evuzZsZUDafukJMeITDxbNTvLr7qatedbVb2sJO2VlH0iClCtei3GTJwW6m6ErTfmbw11F8Je94YVQt2FsPaPviWPfZqdRXTD6/yqe3jxayklbrCELOAaYzxMQLwzM2oB1xjjXQJERIa6F36zgGuM8TYp0TRwUFnANcZ4mE0pGGNM8NgI1xhjgkCwEa4xxgSH2AjXGGOCxq5SMMaYYLAvzYwxJjgEm1IwxpigsRGuMcYEg7emFLzTU2OMOZUAkZH+bYUdSiRGROaJyBIRWSEiT7rldURkroisFZGPRKSsWx7tPl/n7q9dWBsWcI0x3ibi31a4I0AXVW0OtAAuE5F2wP8Bw1S1PpAG3ObWvw1IU9VzgGFuPZ8s4BpjPMydUvBnK4Q6DrpPy7ibAl2AT9zy0UAv93FP9znu/q4iviO7BVxjjLeV3ggXEYkUkcXAbmAKsB7Yr6rZbpVtQHX3cXVgK4C7Px3wmQTZvjQzxnib/1+apYjIgjzPR6rqyLwVVDUHaCEi5YHxQON8jqO5LfvYly8LuMYY7yrC6BVIVdU2/lRU1f0iMg1oB5QXkSh3FFsD2OFW2wbUBLaJSBSQBOzzdVybUjDGeFtEpH9bIUSkojuyRURigW7AKuAH4M9utX7ABPfxRPc57v7vVdVGuMaYM1WpXodbFRgtIpE4g9GPVXWSiKwExonIv4GfgXfc+u8A74vIOpyRbe/CGrCAa4zxtlK6tVdVlwIt8ynfAFyQT/lh4NqitGEB1xjjXZYP1xhjgsVbt/ZawDXGeJvlwzXGmCCx9IzGGBMEYlMKxhgTPDbCNcaY4CgkX0xYsYBrjPEsZ4UdC7jGGBN4IkiEBVxTAv+b9BNfTJmPAld0a8N1V3Tg3XFT+eK7+ZRPjAOg//WX0r51w9B2tASub1WdplUSyDiSzTNT1522PyYqgn7n1yA5tgyREcLUtanM2by/RG2WKxPJrRfU5Ky4Muw7dIx35m0h69hx2tRM4pIGFQE4kn2cjxbvYHv64RK1VRpGvvMFixevIzExjqFP9z9t/8JFa/jksxmIQGRkBDf0vZSGDWqWqM2DB7N4dcR49qTup2JKee658yri4mL58aflTPpqNgAxMWW4+abunF2rconaKi02wnWJyFXAZ0BjVV0dyLbOFBs27+KLKfMZ+dxAoqIiefBfo08E1ut6dKBPr4tC3MPSMWdzGtM37OWm1jXy3X9xvQr8euAIb87eQnzZSB6/tD7zt6ST4zs3CAD1U+Joe3Z5Pli4/aTySxqmsGbPQab8mMolDVK4tEFFJqzYxd5DR3lpxgayjh2nSeV4+rSsxgvTNpTKeZbExR2bc0nXNrz51hf57j+3SR1atWyAiLBl6y5eeW08zw+9w69jr1y1mZmzljLg9itOKv/iy59o0rg2V/a4kImTfuKLL2fT+7ouVKxYnscG30BcXCxLlq7j3VFf8eQTt5T4HEuDlwJuoK+n6APMwo+kDoVxE0qc8TZv302TBjWJiS5LVGQkLZrUZsbclaHuVqlbvzeTzKM5BVdQJbqM8/GMjoog82gOx91g27V+Cg91rsvgrudweeNKfrfZrGoic7c4o+S5W/bTrFoiABv3ZZF17Lj7OJPysWWKc0qlrlHDWsTHxRa4Pyam7Ilgc+TIsZO+rJ/01Wwef/JdBj/2Fp+On+53mwt//oWLOp4HwEUdz2PBojUANKhfgzi3L+fUq86+fQeKejoBIyJ+beEgYAFXROKBDjjr/vR2yz4Skcvz1BklIte4WdafF5H5IrJURAa4+zuJyA8iMhZY5pZ9LiIL3UXe+uc51m0i8ouITBORt0TkVbe8ooh86h57voh0CNQ5l4Y6tSqzZOUm0jMyOXzkKHMW/cLu1HQAPvt6Dv3uH86zr35KxsGsEPc0sKZv2EeVhGievrwhj3Q7h0+W7kSBRpXiqRRflud/2MDQqeuoVT6GehXK+XXMhOgoDhx2EvcfOJxNQvTpf+BdWDuZlbsySvNUAmr+wtU8NOgNXhj2Ebff1gOAZcs3sGvXPp564haefuqvbNz0K6vXbPHreAfSD5FcPgGA5PIJHDiQeVqdaTOW0KxZvdI7iZKQImxhIJBTCr2Ab1T1FxHZJyKtgHHAX4Cv3JUvuwIDcYJyuqqeLyLRwI8iMtk9zgVAU1Xd6D6/VVX3ufkq54vIp0A08DjQCsgAvgeWuPVfxlkAbpaI1AK+Jf8s7mGhdo1KXH/Vxdw/5F3KxUZzTu0qREZG0OuytvS7tjMi8PaH3/HqqK8YfPc1oe5uwDSuFM+2/YcZPnMTKXFlubtjbdanrqNxpXgaVYpnUBfnFz46KoJK8dGs35vJg53qEhUhREdFUK5s5Ik6E5bvYtXug76aA5ypiPZnJzNsxsZC64aL81s34vzWjVi9ZguffDadwf+4nmXLN7Bs+UYefeJtAA4fOcavv+6jUcNa/POp9zh2LJvDR45x6FAWjzz+FgC9r+tCs/MKD6IrV21i+ozFPP7oTQE9L38J4TN69UcgA24f4CX38Tj3+ePAcDeoXgbMUNUsEbkUaCYiuUl+k4D6wFFgXp5gC3CvOzcMTrb1+kAVYLqq7gMQkf8BDdw63YAmef6nJIpIgqqeNoxxR8z9AapUK9mXDyXRo1sbenRzEtO/+cFkKlVI5Kzy8Sf2X3HJ+Tz89JhQdS8o2tVOZsqaPQCkHjrK3kNHqZwQDQKTf9nDjxvTTntN7rxrQXO4GUeySYxxRrmJMVFkHMk+sa9aYjR9W1VnxE+bOORrqiNMNWpYi92708jIyEQVruhxIV07tzqtXu68a0FzuIlJcaTtzyC5fAJp+zNITPztr4ctW3fx9rtf8tADvUmI9++vimCIiPDOnWYB6amIVMBZ6fJtEdkEPIQzsj0CTAP+6D4fl/sS4B5VbeFudVQ1d4R7KM9xO+EE0PbuUsY/AzH4/oMhwq2fe+zq+QVbAFUdqaptVLVN8lk+14ILqLT9zmhs1579zJi7gm4XNSc1z5zZjLkrqRMm3xAHSlrmURpWcv6RSYiOpHJCNKmHjrJq10Han51M2Ujno5sUE0V8tH/T+8t2HqBtrfIAtK1VnqU7nfc0ObYMt7erxZgFW9l98GgAziYwft21j9wFBjZu2kl2dg7x8bE0O68uM2Yu4fBh51z2pR0g/cAhX4c6oVWLBsyctQyAmbOW0bqlM25J3ZvOS698yh39e1K1Suh+N/LjpTncQI1w/wyMUdUBuQUiMh3oiBNk/wq0AW52d38LDBSR71X1mIg0AE4enjiScNaBzxSRRjjrDQHMA4aJSDLOlMI1uHO+wGTgbuB5tx8tVHVxqZ1pADz2/FjSMzKJiozk/tuvJCE+lpdensS6jTtBoGrFZB68o2eou1kiN59fg/oV44gvG8W/ujfkq5W7cWMoszam8c3qPdzQugaPdD0HgAnLf+XQ0RxW7z5IlYRoHuxUF3Au4xq9YBsHjxQ+Kp3ySyq3XlCT9rWTScs8xjtztwLQvXFF4spG8ZcW1QA4rvDcD+sDcNZF8+qI8axavZmDB7O45/7hXNPrYnJynPPs2qU18xesZtaPy4iMjKBs2TLcfefViAjnNa3L9h2pDPn3KABiossycEBPktxLCn25okd7XnltPNNnLqbCWUnce9fVAIyfMJODB7MYNeZrwLkM7V9DbgvMiRdFGM3P+kMKWYKneAd1Fl8bqqrf5Cm7F2fu9F7gV2Ciqt7i7osA/g1cgfP27cGZA24JPKiqPdx60cDnOMsTrwEqAkNUdZo7HfAgzgJvq4B9qvqoiKQAr7ltR+FMYxR67UyT81rqmInTSvhOnLnemL811F0Ie90bhtdIMNz8o+9lrF+xpEThMiqlrpbv8YxfdfeO7rPQ30UkAyUgI1xV7ZRP2fA8Tyucsu848Ii75TXN3XLrHQG6F9DsWFUd6a6eOR5nZIuqpuJMXxhjzjD2pVnoDBGRbjhzupNxRsLGmDOc3dobAqr6YKj7YIwJMvHWnWZnTMA1xvw+WcA1xpggsYBrjDFBYF+aGWNMMHkn3lrANcZ4mHjr1l4LuMYYT7MpBWOMCRbvxNuAJyA3xpiAKq3kNSJS082/vcrNt32fWz5ERLaLyGJ3y5vTe7CIrBORNSLyx8LasBGuMcazSjkTWDbwgKouEpEEYKGITHH3DVPVF05puwnO4grnAtWA70SkgaoWmEnJRrjGGE8rrRGuqu5U1UXu4wycJFjVfbykJzBOVY+4ObvX4SyYUCALuMYYT5MI8Wsr0jFFauNkK5zrFt3tLv/1rpsGFpxgnDdt3jZ8B2gLuMYYbyvCCDdFRBbk2U5fe54T6zF+CvxNVQ8AI4B6QAtgJ/BibtV8Xu4z363N4RpjvKtoyWtSC8uHKyJlcILtf1X1MwBV3ZVn/1vAJPfpNpxlvnLVwMnHXSAb4RpjPEsAEf+2Qo/lRO53gFWq+p885VXzVLsKWO4+ngj0FpFoEamDs77iPF9t2AjXGONhpXqVQgfgRmCZiOQuw/UI0EdEWuBMF2wCBgCo6goR+RhYiXOFw12+rlAAC7jGGI+LKKUE5Ko6i/znZb/y8Zqngaf9bcMCrjHGu/ycLggXFnCNMZ4llN4INxgs4BpjPM1GuMYYEySWLcwYY4LB5nCNMSY4BLEE5MYYEyw2wjXGmCCxOVxjjAkGm8M1xpjgcHIpeCfiWsA1xniah+KtBVxjjLfZnWbGGBMMRcuHG3IWcAsQWzaSpjWTQt2NsPXfqx8NdRfC3rDZw0PdhbD2dEyZEh8jNx+uV1jANcZ4WKnmww04C7jGGE/zULy1gGuM8TCxL82MMSYo7DpcY4wJIgu4xhgTJB6KtxZwjTHeZiNcY4wJBkteY4wxweEkIPdOxLWAa4zxtAgPDXEt4BpjPM1D8bbggCsiib5eqKoHSr87xhjjPzmDktesABTn2uJcuc8VqBXAfhljjF88NIVbcMBV1ZrB7IgxxhSHl74082t9YRHpLSKPuI9riEjrwHbLGGMKJzhXKvjzX6HHEqkpIj+IyCoRWSEi97nlZ4nIFBFZ6/5MdstFRIaLyDoRWSoirQpro9CAKyKvAp2BG92iTOCNQntvjDFBECH+bX7IBh5Q1cZAO+AuEWkCDAKmqmp9YKr7HKA7UN/d+gMjCu2rH524UFUHAIcBVHUfUNav7htjTCCJkw/Xn60wqrpTVRe5jzOAVUB1oCcw2q02GujlPu4JjFHHHKC8iFT11YY/AfeYiETgfFGGiFQAjvvxOmOMCTgR/7aiHVNqAy2BuUBlVd0JTlAGKrnVqgNb87xsm1tWIH+uw30N+BSoKCJPAtcBTxah78YYExBCkW58SBGRBXmej1TVkacdUyQeJ+b9TVUP+Bgd57dDfXWg0ICrqmNEZCHQzS26VlWXF/Y6Y4wJhiJcpZCqqm18VRCRMjjB9r+q+plbvEtEqqrqTnfKYLdbvg3IezVXDWCHz7762dFI4BhwtAivMcaYgPJ3OsGfQbA4Q9l3gFWq+p88uyYC/dzH/YAJecpvcq9WaAek5049FMSfqxQeBT4EquFE8LEiMrjw7htjTOBFiPi1+aEDztVYXURksbtdDgwFLhGRtcAl7nOAr4ANwDrgLeDOwhrwZw73BqC1qmYCiMjTwELgWX/OwBhjAqm0bntQ1Vk+Dtc1n/oK3FWUNvwJuJtPqReFE9WNMSbkzohcCiIyDOcbt0xghYh86z6/FJgVnO4ZY0zBnKsUQt0L//ka4eZeibAC+DJP+ZzAdccYY4pAzpAE5Kr6TjA7YowxxXFGTCnkEpF6wNNAEyAmt1xVGwSwX8YYUyivTSn4c03tKOA9nHPrDnwMjAtgn4wxxm+llUshGPwJuOVU9VsAVV2vqo/hZA8zxpiQEz+3cODPZWFH3Dsw1ovIHcB2fkveYALg8JFj/Kn/Sxw5lk1Odg5Xdm3J4AF/YvP2VG579D3SDmTSvGFN3njqJsqW8eaydNFlo/hy5N+ILhNFZFQkE6f+zNCRX51U586+XbixZ3tyco6Tuv8g9zz1AVt/TStRu+UTy/HuM7dSq+pZbNm5j1sGv0N6RhbXXtaG+266BIBDWUd4YOhHLF+7vURthdLhI8foOfBl5zOUc5wenVvw8O2XM2P+Gp58dQLHVYmLjWb4Y9dTt2bFUHe32EQg0kNzCv6McO8H4oF7ce7EuB24NZCd8kVEctw7QJaIyCIRubCIrx8iIg8Gqn+lIbpsFBNG3MussYOZMXYwU2evZP6yjQx5dQID+3Zm4Wf/JCkxlvcnzA51V4vtyNFseg4czkXXD+Xivs/StX0T2jStfVKdpWu20uWm5+jY91kmTv2ZIff2yv9g+ejQqj6v/fOG08rv73cJM+avoc01TzFj/hru73cpAJt37OVPA16iY99nef6dbxj2SJ8SnV+oRZeN4tNX72Ha+4P4fszD/DBnFQuWb+Qfz3/MiCdv4ocxD3P1pa0ZNurbUHe1xM6oKQVVnauqGaq6RVVvVNUrVfXHYHSuAFmq2kJVmwODOQPveBMR4stFA3AsO4dj2TmICDPm/0LPLi0B6POntnw1fUkou1lih7KOAlAmKpIyUZE4N+78ZtbCtWQdOQbA/GWbqF6p/Il999zQlamjH2LW2MEM6n+53212/0MzPpw0F4APJ83l8k7NAJi3dCPpGVluWxuplqctLyroMyQiZBw6DEDGwSyqpCSFspulIhDpGQPF140P4/GRakxVrw5Ij4omEUiDEynVJgDJQBngMVWd4O57FLgJJ3flHpxbk8NaTs5xOt34f2zctofbrr2YOjVSSEqIJSoqEoBqlZLZsTs9xL0smYgIYdr7D1OnRkXe+d8MFq7YXGDdG3u2Z8pPKwHo3LYRdWtVomu/5xERPnxxABe2rMdPP68vtM1KZyWwa6+z4PSuvQeomJyQT1sX8p3blpfl5Byn2y3Ps3HbHm695iJan1ubYYP70PfvbxATXYaEuBi+fvvvoe5miQh+50kIC74mAF8NWi+KJlZEFuNcolYV6OKWHwaucvNXpgBzRGQi0ArojZNMOApYhAcCbmRkBDPHDiY9I5MbHnqLNRt/Pa2Ohz5n+Tp+XLn4+qEkxsfywfO307heVVatPz3Z0nXdz6dF41r0GPAyAJ3bNaZL20bM+K+z0klcbDR1a1bip5/XM+W9B4kuG0VcbDTJieVO1BnyygS+n7Oq0D51bF2fG65sT/fbh5XimYZGZGQEP4x5mPSMTG4e9Dar1u/gjXE/MPY/d9D63Nq8+sFUnnh5PMMe6RvqrhZfGI1e/eHrxoepwexIEWSpagsAEWkPjBGRpjhfRD4jIhfjrEhRHagMXASMz5N8Z2JBBxaR/jhrE1GzVnisAp+UUI6OreuzYPkm0jOyyM7OISoqkh2706ha0ft/DgIcOJjFrIVr6dq+yWkB9w8XNOTvt/yRHgNe4uixbMD5BRs2ajKjxp8+s3XJLS8Azhxu3yvacteTH5y0f/e+DCpXSGTX3gNUrpDInrSME/vOPacawx/ry7X3jSAt/VBpn2bIJCWU48JW9Zk6exUr1m2n9bm1AejVrSW97y90Ga6wFy7zs/7wdG5bVZ0NpAAVgevdn63dgLyL327U8JmFPc/xRqpqG1VtUzEldN/cpqZlkJ6RCUDW4aNMm7eGBrUrc1GbBkz4/mcAPvxyLt0vbhayPpZUhfLxJMbHAhATXYZOFzRk7aZdJ9U5r0ENhg3uTd8H3iQ17eCJ8u9nr+L6K9sTF+ssrVe1YhIpyfF+tfvNjGX06dEWgD492vL19KUA1KiczJjnbueOf45h/Zbdvg7hCad+hmbMdz5DGQcPnzi/6fPWUL92lVB2s8QEiBTxawsH3rymyCUijXCSo+8FkoDdqnpMRDoDZ7vVZgCjRGQozvleAbwZiv7669fUA9w55H1yjh/n+HHlqm6tuOyi82hUpyq3PfoeT4+YRLOGNbmxZ/tQd7XYqqQk8vqQG4mMiCAiQhj/3SK+nbWcwQP+xOJVW/h6xjKeuq8XcbHRjBp6GwDbfk2j7wNv8sPc1TSoU4XJ7zoXmxzMPMKAJ0afFJQLMmz0FN579lZuuLI923alcfMg5w72h/7anbOS4njh4b8AkJ19nC79ngvQ2Qferr0HuOepD8g5rqgqV3ZpwaUdm/LioN7cOvgdJEIon1COlx718HSCy0NXhSGnfjNcYEWRaFU9EuD++NOPHGBZ7lPgEVX90p23/QLnC7PFOJewdVfVTXm+NNuMsyzGSlV9wVc7rVu30R/nLvBV5Xct+fy7Q92FsLd79vBQdyGs/aHDBSxauKBE4bLyOU31+v984lfdYT0bLyxsiZ1A8yeXwgU4y04kAbVEpDnwV1W9J9Cdy4+qRhZQngrkO+RT1adx8kEYY84gziVf3hni+jOHOxzogfNnO6q6BLu11xgTJiLEvy0c+DOHG6Gqm0/5VyQnQP0xxpgi8dAA16+Au9WdVlARiQTuAX4JbLeMMaZwAkR5KOL6E3AH4kwr1MK51Oo7t8wYY0LOQ/G28ICrqrtx7tQyxpiwIv4vgR4W/LlK4S3yuXFAVfsHpEfGGFMEHoq3fk0pfJfncQxwFU4SGGOMCblwuQLBH/5MKXyU97mIvA9MCViPjDHGT4K3EpAX59beOvx226wxxoROGF1j6w9/5nDT+G0ONwLYBwwKZKeMMcZfEjYrlhXOZ8B11zJrjrOOGcBx9Tf5gjHGBNgZtUy6G1zHq2qOu1mwNcaEldK6tVdE3hWR3SKyPE/ZEBHZ7q6juFhELs+zb7CIrBORNSLyR7/66kedeSLSyp+DGWNMsJXiIpKjgMvyKR/mrqPYQlW/cttsgnN/wrnua15378T1ydeaZlGqmg10BG4XkfXAIZxRvKqqBWFjTEg5y6SXzrFUdYaI1Pazek9gnJuydqOIrAMuAHwupe1rDnceznpg/q9NbYwxQRaEO83uFpGbgAXAA6qahrOE15w8dba5ZT75+rdBAFR1fX5bCTpvjDGlIvdLMz/ncFNEZEGezZ+7ZUcA9YAWwE7gxTxNn6rQ77h8jXArikiBayir6n8KO7gxxgRaEQa4qUVd8UFVTyy056Y5mOQ+3QbUzFO1BrCjsOP5GuFGAvFAQgGbMcaEmBDh51aso4tUzfP0KiD3CoaJQG8RiRaROkB9nGlYn3yNcHeq6lPF6qUxxgSBUHrJa0TkQ6ATztTDNuCfQCcRaYEzXbAJGACgqitE5GNgJZAN3KWqhS7M4CvgeuhyYmPM75JAVCnd+aCqffIpfsdH/SKvlegr4HYtyoGMMSbYSnOEGwwFBlxV3RfMjhhjTHGcUQnIjTEmnHko3lrANcZ4l+BffoJwYQHXGONdYlMKxhgTFM6dZhZwjTEmKLwTbi3gGmM8zkMDXAu4xhgv8zvXbViwgGuM8Sy7SsEYY4LIvjQ7A+SociDrWKi7EbYmfTgk1F0Ie22f+i7UXQhrG3ccKPlBBJtSMMaYYLApBWOMCSIb4RpjTJB4J9xawDXGeJgAkTbCNcaY4PBQvLWAa4zxMkE8NKlgAdcY42k2wjXGmCBwLgvzTsS1gGuM8S6xEa4xxgSN3dprjDFB4CQgD3Uv/GcB1xjjaXaVgjHGBImHZhQs4BpjvM1GuMYYEwQ2h2uMMcEi4qmrFLyUStIYY04jfm6FHkfkXRHZLSLL85SdJSJTRGSt+zPZLRcRGS4i60RkqYi08qevFnCNMZ7lTCmIX5sfRgGXnVI2CJiqqvWBqe5zgO5AfXfrD4zwpwELuMYYTyutEa6qzgD2nVLcExjtPh4N9MpTPkYdc4DyIlK1sDYs4BpjvK20Im7+KqvqTgD3ZyW3vDqwNU+9bW6ZT/almTHG04rwpVmKiCzI83ykqo4sZrP5NaqFvcgCrjHG04oweE1V1TZFPPwuEamqqjvdKYPdbvk2oGaeejWAHYUdzKYUjDHeFtgphYlAP/dxP2BCnvKb3KsV2gHpuVMPvtgI1xjjWU4sLZ3rcEXkQ6ATztTDNuCfwFDgYxG5DdgCXOtW/wq4HFgHZAK3+NOGBVxjjHeVYj5cVe1TwK6u+dRV4K6itmEB1xjjad65z8wCrjHG0wTx0K29FnCNMZ7moXhrAdcY410luwAh+CzgGmO8zUMR1wKuMcbTLAG5KbEO1z1FXGwMkZFCZGQEk956gP0HDnHXkDFs27mPGlXP4vUn+5GUUC7UXS2Wl974nPk//0JSYhyvP3/61TVbt+/hpTcnsH7jTm76Sxeu7tGhxG0eO5bNf14fz7qNO0iIL8fD9/2ZyhWT+XnpekaN+47s7ByioiK5te8lNG9at8TtlUTlxGieuuY8UuLLclzhswXb+HDOlnzrNqmWyOj+bRn08VKmrtxVonYTY6MYel1zqpWPYcf+wzz80RIyDmfTvVkVbu5YB4DMozk888VK1u46WKK2SouX5nDD9k4zEaksImNFZIOILBSR2SJyVSkcd5qIFPX2vpAY9/KdfP3uQ0x66wEAXv/vVDq0qs/0Dx+lQ6v6vP7B1BD3sPi6/aEFTw66ocD9CfGxDOjXnat7XFjkY+/ak8agp947rXzyD4uIi4vhrZfuo+fl7Rg19jsAEhPK8cSDfXjtuTu5f2AvXnx9fJHbLG05x5Vh36zhmld+ot/IuVx3QU3qVIw7rV6EwH2XNmD2utQiHb917WSGXHXuaeW3XFSHeRuI3Z1hAAAPl0lEQVT20uvlH5m3YS+3XOQE2e1pWfz13fn85fXZvDV9A4/1PP21IeFeh+vPFg7CMuCKc53H58AMVa2rqq2B3jj3K/9uTZm1nGsuOx+Aay47n8mzloW4R8XXtHFtEuJjC9xfPimeBvWqExl5+kf0h5lLuP+xkdwzaASvvv0FOceP+9XmnIVr6HpxCwA6tm3CkuUbUFXq1alKhbMSATi7RiWOHcvm2LHsYpxV6Uk9eJTVOzMAZ0S5cc8hKiVGn1avd7taTF25i32Hjp5UflOH2rw/oC0f3dmeOzrX87vdPzSqxKSfnZQAk37eQafGTnKspVvTyTjsvCfLtu6ncj59CRXx879wEJYBF+gCHFXVN3ILVHWzqr4iIjEi8p6ILBORn0WkM4CP8lgRGedmZf8IKPi3PKwINzzwBn/664uMnfgTAKlpGVROSQKgckoSqWnh8SddMG3dvocZc1bw/JDbeGXoQCJEmDZrqV+v3bvvABUrOIE1MjKScuViOJCReVKdH+etpG7tKpQpEz6zbVXLx9CwagLLt6WfVF4xIZrOjSvxyfytJ5W3q1eBWhXKceObc+k9YjaNqyXS6uxkv9qqEFeW1INO8E49eJSz4sqeVqdX6+r8uLZoI+pAEbw1wg2fT9XJzgUWFbDvLgBVPU9EGgGTRaSBj/KBQKaqNhORZj6OG1Y+e/1eN6hmcMPf36Bercqh7lJYWLx8A+s37OD+x5ysekePZpOU5Pyp/e8Xx7FrTxrZ2TnsSU3nnkFOEv4rL2vHJZ1a5ps8L+9F85u37mbU2O/41yM3Bv5E/BRbNpIXerfgxa/XcOhIzkn7HuzekOGT13L8lPNqd04F2tWrwIcD2wFQrmwUNSuUY9HmNEb3b0vZSKFc2SgSY8vw4cAEAIZPWcvsdXsL7U+bOsn0alWdW9+eXzonWArCJJb6JVwD7klE5DWgI3AUJy3aKwCqulpENgMN3P35lV8MDHfLl4pIgcMhEemPs1wGNWrWCtj5+CN3JJuSnMAfLzqPxau2kJKcwK7UdCqnJLErNZ2U5PiQ9jEkFLpc3IKb+3Q7bddjD/QGnDncYSM+Z+gTJ+cTqVAhkT17D5BSIYmcnBwyMw+fmNZI3ZvO0/8Zx9/vvIqqlc8K/Hn4ISpCeKF3c75aupPvV+0+bX+T6kk8e20zAMqXK0PH+hXJOX4cAd6buZFPF2w77TX9Rs4FnDncK1pWY8j4FSft33voKCnxzig3Jb7sSVMV9SvH83jPc7nn/UWkZx0rxTMtIQ9F3HCdUlgBnFiUTVXvwkkgUZGC315fb3uhiYHddkaqahtVbVMhJcXfvpa6zKwjHMw8fOLxjPlraFi3Ct06NOXTb5yRxaffzOeSjk1D1sdQad60Dj/OW8n+dGc6JeNgJrv37PfrtW1bN2TqjMUAzJq7kmbn1kFEOHgoiyHPjaVf7240aRjaf2jzeqLXuWzcc4j//rQ53/1XDJtJD3f7buUunp20immr9zB73V6ubFWd2LKRgDP1kJzP1EB+ZqzeQ4+W1QDo0bIa01c7gb5KUgwv9G7B458uY8veTF+HCLpSXNMs4MJ1hPs98IyIDFTV3MXZcq9/mgFcD3zvThnUAtb4Uf6DiDQFmgXvNIonNS2D/o8637Jn5+TQs1trOrVtTPNGtbjzn6P56Mu5VKuczIin+hVypPD13PBPWLZqEwcyMul314tc/+fOZGc7fzJffsn5pO3P4G+PjiQz6wgRIkz4eg4jnr+LWjUqceN1XXj82ffR40pkVCQDb7mcShXLF9rmpZ1a8uLr47n9by8THx/Lw/f8GYBJ385j5659jBs/nXHjpwPwr8E3Uj4pdH9BtKhVnh4tqrH214wTUwOvfreOKkkxAPmOXnPNWb+XOhXjGHX7BQBkHc3hsU+WkXao8Hbfm7mR//tLM3q1qs6v6Yf5x0dLALi9U12SypVhcI/GgHMVxQ1vzi3JKZaa8Ail/hEny1j4cbOrDwPaAnuAQ8AbOAmA3wBaA9nA31X1BxGJKaA8FngPaAIsBs4B7lXVBfjQolVrnTx9TkDO7UywbHt64ZV+5+4Z44mvC0Jm43t3k7XzlxLFy6bNW+lnk2f5VbdhlbiFxVjxoVSF6wg3d8G23gXsvjmf+ocLKM/ycRxjjIeVZgLyYAjbgGuMMYUKo0u+/GEB1xjjaR6KtxZwjTFeZgnIjTEmaDwUby3gGmO8yxKQG2NMMHko4lrANcZ4ml0WZowxQWJzuMYYEwziJGH3Cgu4xhiP807EtYBrjPGs3ATkXmEB1xjjaR6KtxZwjTHeVpojXBHZBGQAOUC2qrYRkbOAj4DawCbgOlVNK87xwzUBuTHG+EVE/NqKoLOqtsiTynEQMFVV6wNT3efFYgHXGONp4udWAj2B0e7j0UCv4h7IAq4xxrP8XbG3CANcxVmAdqG7xiFAZTc/d26e7krF7a/N4RpjPK0Id5qliEjelV5GqurIU+p0UNUdIlIJmCIiq0ulky4LuMYYb/N/9Jpa2BI7qrrD/blbRMYDFwC7RKSqqu50l/46fQllP9mUgjHG00prDldE4kQkIfcxcCmwHJgI5K7Y2g9nXcVisRGuMcbDSnUJ9MrAePeKhihgrKp+IyLzgY9F5DZgC3BtcRuwgGuM8azSvNNMVTcAzfMp3wt0LY02bErBGGOCxEa4xhhPs1wKxhgTJJaA3BhjgqFoNzWEnAVcY4xnWXpGY4wJIptSMMaYILERrjHGBImH4q0FXGOMx3ko4lrANcZ4lkBp3tobcKKqoe5DWBKRPcDmUPcjjxQgNdSdCHP2HvkWbu/P2apasSQHEJFvcM7LH6mqellJ2ispC7geISILCkst93tn75Fv9v6EnuVSMMaYILGAa4wxQWIB1ztOXQrEnM7eI9/s/Qkxm8M1xpggsRGuMcYEiQXcABKRq0RERaRRqPvidSKSIyKLRWSJiCwSkQuL+PohIvJgoPoXCiJSWUTGisgGd1nv2SJyVSkcd5qI2NUMAWABN7D6ALOA3iU9kIhElrw7npalqi1UtTkwGHg21B0KJXEW3vocmKGqdVW1Nc7nrEZoe2Z8sYAbICISD3QAbsMNuCLykYhcnqfOKBG5RkQiReR5EZkvIktFZIC7v5OI/CAiY4Flbtnn7mhmhYj0z3Os20TkF3d08paIvOqWVxSRT91jzxeRDsF7FwImEUgD530WkanuqHeZiPTMrSQij4rIGhH5DmgYqs4GSBfgqKq+kVugqptV9RURiRGR99z342cR6QzgozxWRMa5n72PgNjQnNKZz27tDZxewDeq+ouI7BORVsA44C/AVyJSFmdhuoE4QTldVc8XkWjgRxGZ7B7nAqCpqm50n9+qqvtEJBaYLyKfAtHA40ArIAP4Hlji1n8ZGKaqs0SkFvAt0DjA5x4IsSKyGIgBquIEHIDDwFWqekBEUoA5IjIR573oDbTE+ZwvAhYGv9sBcy7OOeXnLgBVPc+dzposIg18lA8EMlW1mYg083FcU0IWcAOnD/CS+3ic+/xxYLgbVC/D+XMwS0QuBZqJyJ/d+klAfeAoMC9PsAW4N888XU23XhVguqruAxCR/wEN3DrdgCby2/3miSKSoKoZpXu6AZelqi0ARKQ9MEZEmuLcTv+MiFwMHAeq4yx3fREwXlUz3ddMDE23g0NEXgM64nxmtgGvAKjqahHZjPN56FhA+cXAcLd8qYgsDf4Z/D5YwA0AEamAMwJrKiIKRAIK/AOYBvwRZ6T7Ye5LgHtU9dtTjtMJOHTK825Ae1XNFJFpOCM+X9k7Itz6WSU9r3ChqrPd0WxF4HL3Z2tVPSYim3DeE3De8zPVCuCa3Ceqepf7niwAthfwGl+fkzP5vQobNocbGH8Gxqjq2apaW1VrAhtxRhjjgFtwRmC5AfZbYKCIlAEQkQYiEpfPcZOANDfYNgLaueXzgD+ISLKIRJHnFxGYDNyd+0REWpTaWYaIe+6RwF6c92S3G2w7A2e71WYAV7nzkwnAFaHpbcB8D8SIyMA8ZeXcnzOA68H5LAG1gDV+ljcFmgWh/79LNsINjD7A0FPKPgX6AvcCY4CJqnrU3fc2UBtY5H77vAdnDvhU3wB3uH/yrQHmAKjqdhF5BpgL7ABWAunua+4FXnNfE4Xzy3VHKZxjsOXO4YIzUuunqjki8l/gCxFZACwGVgOo6iL3C6DFOFnfZoai04GiqioivYBhIvIPnM/MIeBhYALwhogsA7KBm1X1iIi8XkD5COA99zOyGOcfcBMAdqfZGUJE4lX1oDvCHQ+8q6rjQ90vY8xvbErhzDHEHQEux5m++DzE/THGnMJGuMYYEyQ2wjXGmCCxgGuMMUFiAdcYY4LEAq4pFvkte9dyEfmfiJQr/FUFHquTiExyH18pIoN81C0vIncWo418s4UVVH5KnVF57gL0p63aIrK8qH00Zz4LuKa4crN3NcW5nfSka3vFUeTPl6pOVNVTr2HOqzxQ5IBrTDiwgGtKw0zgHHdkt8q9wH4RUFNELhUnT+sidyQcDyAil4nIahGZBVydeyARuVl+y3RWWUTGi5MDd4k4OXCHAvXc0fXzbr2H5LdMa0/mOVaRsoWJyO3ucZaIk2Et76i9m4jMFCcjWw+3fr5Z3owpiAVcUyLujRbdcdNH4gS2MaraEufOp8eAbqraCuc+/7+LSAzwFs7tthfhJN/Jz3CcpDzNcbJ/rQAGAevd0fVDbuKf+jhZ1VoArUXkYhHJzQ/bEiegn+/H6Xymque77a3CyeKWqzbwB+BPOHdrxZAny5t7/NtFpI4f7ZjfKbu11xRX3lttZwLvANWAzao6xy1vBzTBSTcJUBaYDTQCNqrqWgAR+QDoz+m6ADcBqGoOkC4iyafUudTdfnafx+ME4ASKni2sqYj8G2faIp7fcl0AfKyqx4G1IrLBPYeCsrz94kdb5nfIAq4prhPpEnO5QfVQ3iJgiqr2OaVeC0ovO5UAz6rqm6e08bditDEK6KWqS0TkZqBTnn2nHkspOMtb7SK2a34nbErBBNIcoIOInAMgIuXcLFWrgToiUs+t16eA10/FSY6dO1+aiJNgPSFPnW+BW/PMDVcXkUoUL1tYArBTnKxt15+y71oRiXD7XBcneZC/Wd6MAWyEawJIVfe4I8UPxUm6DvCYuwpGf+BLEUnFWfetaT6HuA8YKSK3ATnAQDcX7o/uZVdfu/O4jYHZ7gj7IHBDMbOFPY6TcW0zzpx03sC+BpiOk9z8DlU9LCL+ZnkzBrBcCsYYEzQ2pWCMMUFiAdcYY4LEAq4xxgSJBVxjjAkSC7jGGBMkFnCNMSZILOAaY0yQWMA1xpgg+X+PcX41dJL4nAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "\n",
    "tuned_parameters = [{'var_smoothing': [1e-6, 1e-07, 1e-08, 1e-09, 1e-10, 1e-11, 1e-12, 1e-13, 1e-14, 1e-15]}]\n",
    "\n",
    "clf = GridSearchCV(GaussianNB(), tuned_parameters, scoring='f1_macro')\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "#Evaluation\n",
    "\n",
    "print(\"Best parameters set found on development set:\\n\", clf.best_params_)\n",
    "print(\"\\n Best Score\", clf.best_score_) \n",
    "\n",
    "print(\"\\n Grid scores on development set:\")\n",
    "means = clf.cv_results_['mean_test_score']\n",
    "stds = clf.cv_results_['std_test_score']\n",
    "for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "    print(\"%0.3f (+/-%0.03f) for %r\" % (mean, std, params))\n",
    "\n",
    "print(\"Detailed classification report:\")\n",
    "print(\"The scores are computed on the full evaluation set. \\n\")\n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(classification_report(y_true, y_pred))\n",
    "\n",
    "#Lets print confision matrix\n",
    "labels =['Average','Bad','Good']\n",
    "disp = plot_confusion_matrix(clf, X_test, y_test,\n",
    "                                 display_labels=labels,\n",
    "                                 cmap=plt.cm.Blues)\n",
    "\n",
    "print ('\\nTesting score: ',clf.score (X_test, y_test)) # print the testing score(accuracy)\n",
    "print ('\\nTraining Score: ',clf.score (X_train, y_train)) # print the trainig score(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['fit_time', 'score_time', 'test_f1_macro']\n",
      "test_f1_macro [0.48536636 0.51010628 0.50258404 0.52354389 0.51960363]\n"
     ]
    }
   ],
   "source": [
    "#Cross Validate\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.metrics import recall_score\n",
    "\n",
    "scoring = ['f1_macro']\n",
    "scores = cross_validate(clf, X, Y, cv=5, scoring=scoring)\n",
    "print(sorted(scores.keys()))\n",
    "print('test_f1_macro', scores['test_f1_macro'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K best Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 0 0 0 0]\n",
      " [0 1 0 0 0]\n",
      " [0 0 0 0 1]\n",
      " ...\n",
      " [1 0 0 0 0]\n",
      " [0 0 0 1 0]\n",
      " [1 0 0 0 0]] ['Bad' 'Bad' 'Bad' ... 'Average' 'Bad' 'Average']\n",
      "4800 4800 1200 1200\n"
     ]
    }
   ],
   "source": [
    "#Load Kbest data\n",
    "arr2= np.load('data_kBest.npy', allow_pickle= True)\n",
    "X= arr2[:,0:-1]\n",
    "Y= arr2[:,-1]\n",
    "\n",
    "\n",
    "### Split arrays or matrices into random train and test subsets - 20% for Test, 80% to train\n",
    "X_train, X_test, y_train, y_test= train_test_split(X, Y, test_size=0.20,  random_state = 200)\n",
    "\n",
    "print(X_train,y_train)\n",
    "print(len(X_train),len(y_train),len(X_test),len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters set found on development set:\n",
      " {'var_smoothing': 1e-06}\n",
      "\n",
      " Best Score 0.4688739965134472\n",
      "\n",
      " Grid scores on development set:\n",
      "0.469 (+/-0.020) for {'var_smoothing': 1e-06}\n",
      "0.469 (+/-0.020) for {'var_smoothing': 1e-07}\n",
      "0.469 (+/-0.020) for {'var_smoothing': 1e-08}\n",
      "0.469 (+/-0.020) for {'var_smoothing': 1e-09}\n",
      "0.469 (+/-0.020) for {'var_smoothing': 1e-10}\n",
      "0.469 (+/-0.020) for {'var_smoothing': 1e-11}\n",
      "0.469 (+/-0.020) for {'var_smoothing': 1e-12}\n",
      "0.469 (+/-0.020) for {'var_smoothing': 1e-13}\n",
      "0.469 (+/-0.020) for {'var_smoothing': 1e-14}\n",
      "0.469 (+/-0.020) for {'var_smoothing': 1e-15}\n",
      "Detailed classification report:\n",
      "The scores are computed on the full evaluation set. \n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Average       0.46      0.28      0.35       405\n",
      "         Bad       0.53      0.49      0.51       392\n",
      "        Good       0.49      0.72      0.58       403\n",
      "\n",
      "    accuracy                           0.50      1200\n",
      "   macro avg       0.50      0.50      0.48      1200\n",
      "weighted avg       0.50      0.50      0.48      1200\n",
      "\n",
      "\n",
      "Testing score:  0.48276523868797655\n",
      "\n",
      "Training Score:  0.46904203445488185\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEKCAYAAABewe3GAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd8FVXawPHfk5tGKoEk9F5FpFtQVxEb9l5Y165YQNd1LejqK/a2q66KKHZcFF0BZV1XUQGxgdIRKVKlB0gICQmkPe8fMwkXSLkhuWXw+fq5n2TOnDtzZrw8OfeZM2dEVTHGGBN8UeFugDHG/F5YwDXGmBCxgGuMMSFiAdcYY0LEAq4xxoSIBVxjjAkRC7jGGBMiFnCNMSZELOAaY0yIRIe7AZGqYaPG2rRF63A3I2JJuBvgAVsLisLdhIiWt2UDu3bk1Omj5Etpo1pSGFBdLdzyuaoOqsv+6soCbhWatmjNGxOmhLsZEStKLOTW5I2568LdhIg2cfgldd6GlhQS1+XigOrumjcyvc47rCMLuMYYDxMQ72RGLeAaY7xLgChfuFsRMAu4xhhv81B6ywKuMcbDLKVgjDGhYz1cY4wJAcF6uMYYExpiPVxjjAkZG6VgjDGhYBfNjDEmNARLKRhjTMhYD9cYY0LBUgrGGBMaAvjsopkxxoSG5XCNMSYULKVgjDGhYz1cY4wJEevhGmNMCIjd2muMMaFjt/YaY0wo2EUzY4wJHUspGGNMCNh8uMYYEyqWUjDGmNCxi2bGGBMilsM1xpgQEEspGGNM6FgP1xhjQkMs4BpjTPA5T9ixgGuMMcEngkRZwP3de/qlicyYs5SGqYm8/o9b9lv/5TfzGffxNwA0iI/ltuvOokPbZnXaZ1FxCU++OJ5lKzeQkpzA/bddTNPMNGYtWM5rY7+gpKSE6Ohobrj8VHp3b1+nfdXVky9NYMZs5/y8+cyt+63/bf0Wnhw5gV9XbeDawSdzydnH1nmfRcUlPP7ChxXn54G/XOKcn/nLGT12MiUlpURH+7jx8lPpc1iHOu+vrv7YpwWHNk0mb3cJT3y1fL/18dFRXNGvJWkJMUSJMOXXrcz8bXud9pkQ4+OqI1rRKCGG7IJi3vzxNwqLy+jXMpUTO2cAUFRSxvvzNrBhx6467au+eKmHG9TLeyJynoioiHQN5n4i0akDevP4vVdUub5ZZhrPjriW1/4+jD9dMIBnRk8KeNubsnK4fcTr+5X/b8pskhIb8M4Lf+GCM/rz6tjJAKQmJ/DI3Zfx2j9u4e6h5/P4Cx/W/oDq2aABvXnyb1dWuT45qQG3XHMGF59V+0C7KSuH2x54bb/yT6fMJjmpAWNfvJ2LzjyaV/71OQCpKQk8NvxPvPHMLdwz7IKIOD8AM9fkMOq71VWu/0P7xmzK282TU1bwwjerOPewpvgCDD4d0xO5rE+L/cpP6pzOsi35PPLFryzbks/JbpDdVlDE89+s5Mkpy/lsaRaX9m5+QMcUDCIS0CsSBHs8xWDgW+DSum5IRLwzuhno0a0tKUkNqlx/aJfWJLvru3VqxZZtuRXrvpg+j5vveZkhd47kmdEfU1pWFtA+v5+1hFMG9ALg+KMOZc7PK1FVOrVrTnqjFADatsqkqLiEouKSAz20etGzW7tqz09aahJdO7YkOnr/j+gX0+dx0/BRXHfHi/zjlY8oLQ3s/Hz302JOPb43EPnnB2DFtgIKikurqaHEuecnNjqKgqJSylQBGNgpnb8OaM/dAztyWtfMgPd5WLMUflzj9JJ/XLOdw5o552VVdiGFxc55Xp1dQMMGMQdwRMFRXwFXRFqJyFQRWSwii0Tkz275CBFZLyLz3Nfpfu+5R0SWi8hSETm1pn0ELeCKSBJwDHAtbsAVkff3aexbInKBiPhE5GkR+UlEFojIDe76Ae4JeBdY6JZ9JCKz3RMyxG9b14rIMhGZJiKvisiLbnmGiIx3t/2TiBwTrGM+UP+bMpsjencGYM26LKZ9/zPPP3w9o58eii8qiq++mR/QdrZm7yCzcSoAPp+PxIQ4duQV7FVn+sxFdGrXjNgYb2aT1qzLYur3C3nhkSG89vdhREVF8eW3tTg/6XvOT1Jl52fGIjp65PxMX5lN0+Q4Hj6tC/ec2JHxCzaiQNfMJDISY/nHtJU8NWU5rdLi6dA4IaBtJsdFs2O388dmx+4SkuP2Pw/926SxeHNefR7KgZNavGpWAvxVVQ8BjgKGikg3d92zqtrLfX0K4K67FDgUGAS8VFPHMJifqnOBz1R1mYhki0gfYBxwCfCpiMQCJwI34QTlXFU9XETigO9EZLK7nSOA7qq6yl2+RlWzRaQB8JOIjAfigPuBPkAeMAUo/1f4T5yT9a2ItAY+Bw4J4nHXytyfV/K/qbN57qHrK5Z/XbWBm+95GYDdRcU0TEkE4P+efpdNWTkUl5SStTWXIXeOBOD80/sz6IQ+qNu72YvfX/bVazfz6tjJPPW3q4J7UEE0Z+FKlq3cwI3DRwFQVFRCWqpzfu5/aiwbs3IoKSll89ZcrrvjRQAuOKM/p53Ql8pOj//5WbV2M6PHfs5T910V7MOoF4dkJrEudxcvfLua9MRYhh7TlienLKdLZhJdM5O46wQnDx0XHUVGUhwrthVw+/HtiY4S4qKjSIj1VdSZtGgzS7Lya9xnp/REjmqbxnPTV9VYNxSE+ksXqOpGYKP7e56ILAb2z7vscQ4wTlV3A6tEZDlOvPqhqjcEM+AOBp5zfx/nLt8PPO8G1UHAdFUtFJFTgB4icqFbPxXoBBQBP/oFW4BbReQ89/dWbr2mwNeqmg0gIv8GOrt1TgK6+f1PSRGRZFXd70+022MeAtCkecs6HXwgVqzZxD9e+YjH77mC1GSnB6IKpxzfi+v+eMp+9R+684+Ak6N86qUJPDPi2r3WZzROJWtbLhmNUyktLWVnwe6Kr+1btuXyf39/j+FDL6B500ZBPrLgUZRTj+/N9Zftf34evusywDk/T4wcz3MPXrfX+ozGKWRt3XN+8vc9P0+/y/BhF9KiaePgH0g9OLJNGl8s2wLA1p1FbCsoIjM5DgG+WLaF71fn7PeeZ75eCTg53CNbN2TsnPV7rc/bXUKK28tNiYsmb/ee1ErzlDgG927BqB9WU1BUXaojtKKi6v+Luoi0BXoDM3G+qQ8TkSuAWTi94BycYDzD723rqD5AByelICKNgYHAayKyGrgTp2e7G5gGnOoujyt/C3CLX5e9naqW93B3+m13AE4A7a+qPYG5QDzVf2GIcuuXb7tFZcEWQFVHq2o/Ve3XsFH6gRx6wDZv3c6Iv7/HPcMupFXzPfvqfVh7ps9YRE6u09vYkV/A5i2BXXnu37crk6fNA+DrGYvofWg7RIT8nYXc+8Q7XDf4ZLp3bVP/BxNCfbp34Gv/85NXwKYt+weWyhzdryuffz0XcM9P9/YV52f44+9w3R9P4TAPnZ+cgiK6ZCQBkBznIzMpjm07i1iSlc9RbdKI9Tn/vFPjo0mKDewSyM+bdnBEm4YAHNGmIQs37gAgrUEM1x7Zmndmr2VLflEQjubA1SKHmy4is/xeQ6rYXhIwHrhNVXcAo4AOQC+cHvA/yqtW8vbKvkdVCFYP90JgjKreUF4gIl8Dx+IE2euAfsBV7urPgZtEZIqqFotIZ2DvP72OVCBHVQvckQ9HueU/As+KSBpOSuEC3JwvMBkYBjzttqOXqs6rtyOtwiPPfcD8X1aRm1fAJTc+zZUXD6S0xOkVnHXKEbzz4TR25Bfwz9f+A4DPF8WoJ26ibctMrr70JO5+5G3KVIn2+bj12jNpktGwxn2ePrAPj784nstveZbkpAbcd9vFAHz02Uw2bMrmX+On8a/x0wB48r4rSUtNCs7BB+Dh595n3iLn/Fx0w1NcdfHAiotfZ59yBNk5edwwfBQFhbsRET787/e89eyttG2VyTWXnsSdD7+FquLz+bjturNompFW4z7PGNiXx174kMuGPUNKUgPu/8slAEz8bAYbNm3jnQ+n8s6HUwF4+v6rwnp+AK7s15KOGYkkxUbz0KAufLo4C5/7T/y71Tl8tnQLf+rTkuEDO4LApEWb2FlUypKsfJokx3H78c7Qv92lZbwzax35AfRKv1i2lasPb8VRbdLIKSjmzR/XAjCoawaJsdFc1NMZnVCm8PdpK4Jz4LUReH4WYKuq9qt2cyIxOMF2rKpOAFDVzX7rXwU+cRfX4XzLLtcS2FDt9ivN+9WRiEwDnlDVz/zKbsXJnd4KbAImqerV7roo4BHgLJzTtwUnB9wbuENVz3TrxQEf4XTblwIZwAhVneb+tboD54AXA9mq+jcRSQdGuvuOxklj3FjTMXQ9rLe+MWFKXU/FQSsqQobZRLI35q4LdxMi2sThl7BlxaI6fZCi09trwzMfC6jutrcHz64u4IrTDX4bJ3bc5lfezM3vIiJ/AY5U1UtF5FDgXZy8bXPgK6CTqlb5ly0oPVxVHVBJ2fN+i433WVcG3Ou+/E1zX+X1dgOnVbHbd1V1tIhEAxNxerao6lac9IUx5iBTnxfNcHK1lwMLRaT8W/C9wGAR6YWTLlgN3ACgqotE5APgF5wRDkOrC7ZwcN1pNkJETsLJ6U7G6QkbYw5y9XVrr6p+S+UJik+rec+jwKOB7uOgCbiqeke422CMCTHx1q29B03ANcb8PlnANcaYELGAa4wxIVDPF82CzgKuMcbbvBNvLeAaYzxMgnNrb7BYwDXGeJqlFIwxJlS8E28t4BpjvM16uMYYEwKBPs0hUljANcZ4mgVcY4wJEXtMujHGhIj1cI0xJhRs8hpjjAkNYa/ngEY8C7jGGA+zUQrGGBMyUXbRzBhjQkAspWCMMSEhWA/XGGNCxnq4xhgTInbRzBhjQsFyuMYYExqC2ATkxhgTKtbDNcaYELEcrjHGhILlcI0xJjScuRS8E3Et4BpjPM1D8dYCrjHG2+xOM2OMCQWbD/fgEBcdRduMxHA3I2Ld/OGCcDch4g07pm24mxDRpjWIqfM2bD5cY4wJGZsP1xhjQsZD8dYCrjHGw8QumhljTEjYOFxjjAkhC7jGGBMiHoq3eGdeM2OMqYSIBPQKYDutRGSqiCwWkUUi8me3vJGIfCEiv7o/09xyEZHnRWS5iCwQkT417cMCrjHGu9zJawJ5BaAE+KuqHgIcBQwVkW7AcOArVe0EfOUuA5wGdHJfQ4BRNe3AAq4xxrOcCcgDe9VEVTeq6hz39zxgMdACOAd42632NnCu+/s5wBh1zAAaikiz6vZhOVxjjKdFBSGJKyJtgd7ATKCJqm4EJyiLSKZbrQWw1u9t69yyjVVt1wKuMcbTahFv00Vklt/yaFUdvf/2JAkYD9ymqjuqyf9WtkKra0CVAVdEUqp7o6ruqG69McYEm9Ru8pqtqtqv+u1JDE6wHauqE9zizSLSzO3dNgOy3PJ1QCu/t7cENlS3/ep6uItworX/0ZQvK9C6ug0bY0wo1NeNZuJE7teBxar6jN+qScCVwBPuz4/9yoeJyDjgSCC3PPVQlSoDrqq2qmqdMcZEinq8tfcY4HJgoYjMc8vuxQm0H4jItcBvwEXuuk+B04HlQAFwdU07CCiHKyKXAu1V9TERaYmTRJ5dmyMxxpj6JjgjFeqDqn5L5XlZgBMrqa/A0Nrso8ZhYSLyInACTuQHJ5K/XJudGGNMsERJYK9IEEgP92hV7SMicwFUNVtEYoPcLmOMqVmAd5FFikACbrGIROEOdxCRxkBZUFtljDEB8lC8DSjgjsQZJpEhIg8CFwMPBrVVxhgTACE4Nz4ES40BV1XHiMhs4CS36CJV/Tm4zTLGmMAcjBOQ+4BinLSCzb9gjIkItZiYJiIEMkrhb8B7QHOcOyneFZF7gt0wY4wJRJRIQK9IEEgP909AX1UtABCRR4HZwOPBbJgxxgQiMkJpYAIJuGv2qRcNrAxOc4wxpnYOimFhIvIsTs62AFgkIp+7y6cA34amecYYUzVnlEK4WxG46nq45SMRFgH/9SufEbzmGGNMLUhgk4tHiuomr3k9lA0xxpgDcVCkFMqJSAfgUaAbEF9erqqdg9guY4ypkddSCoGMqX0LeBPn2E4DPgDGBbFNxhgTsPp6am8oBBJwE1T1cwBVXaGq9+HMHmaMMWEnAb4iQSDDwna7M6GvEJEbgfVAZg3vMXWw8rcsbn1oTMXy2o3buO3qQcxZtIZVa52ne+zILyQlqQGfvHZHuJpZJzcc05Y+LVPZsauEOz9etN/6xFgfNxzTlibJcRSXKi9/t4p123fVaZ/RUcLQP7SjXeME8neX8M+vV7Ilv4jDmqUwuG8Lon1CSakydtY6Fm3Kq9O+6kOnzCTSEmMpLi1j7m/b91ufkRxHy7QGAJSWKSuy8tlZVFqnfYpA5ybJJMVFU1JWxpKNeewuKaNhQgxtGyciAqqwautOcguL67Sv+iACPg/lFAIJuH8BkoBbcXK5qcA1wWxUdUSkFFiI80erFBimqt/X4v0jgHxV/XtwWlh37VtnVgTS0tIyjr7oQU459jCuvvD4ijqPvfQxyYnxVW0i4n29fCufL85i6B/aVbr+3B7NWJNdwDNTV9A8NZ5rjmzNI5OXBbTtjKRYbjq2HQ99tnSv8hM6pZNfVMJtE36mf7s0/ti3Jf/8eiV5u4t5+qvl5BQW07JhPPee3Jmb/72gzsdYV5t37GJDbiGdmyRXun5XcSkL1uVSWqakJcTQsUkS89fmBrTtuOgoOjdJZuH6ves3SYmnpKyM2WtySE+KpW16Iks35VFcWsYvG3ZQVFpGQqyPQ1uk8NOqnDofY32IlHRBIAKZvGam+2seeyYhD6dCVe0FICKn4tzxdnz1b/Gu7+f8SuvmjWnRtFFFmary32nz+dczN4WxZXWzZHM+GUlVT6vcIjWejxduAmBD7i4ykmJJjY8md1cJx7ZvxKBDmhDtE5Zv2cnrM9ag1T4r1dGvdUM+nOc842/m6hyuPtJ5LN/q7MKKOuu27yLGF0V0lFBSFsBGg2jHrhLioqvO+uXtKtmrbqxf3YzkOJo3jEdEyNtVzIqsnQHts3FiLL9lFwCwNb+IDplJAOzcvafnXFBUSpRIRW833DwUb6u98WEi1TzyV1XPD0qLaicFyIGKRxt/DKQBMcB9qvqxu+5vwBU4z5DfgnNrsid8MmUuZ53Ye6+ynxasJD0tiXYtM8LUquD7LaeQI9o0ZGlWPh3SE0lPiqNRYiyJcdH0b9eIBz5dQqkq1xzVmmPbN+abFdtq3GajhFi27SwCoEyhsKiU5Lho8nbvCVxHtkljdXZB2INtbTVNiSdnp/MVv0GMj4ykOBaszUWBDhmJZCbHkZW3u8btxEZHsbtkz3TXJaW63x+fxkmx7NxdEhnBlsiZJyEQ1fVwXwxZK2qngfuAt3igGTDQLd8FnOc+Rz4dmCEik4A+wKVAb5zjnYNHAm5RcQlffb+IO68/Y6/y/0yZy1kn9glTq0Lj44UbufKI1jxxdjfW5hSyOruA0jLlsGbJtGucwKNnHQJArC+KHW5P7/YTOpCZHEd0lJCeGMsTZ3cD4H+/bObr5ZUHZPXrU7RsGM8f+7bgsS9+DfLR1a/UBjE0SXUCLEDDhBgS4330bN0QcIZNFZc6x3lIs2TiYnxEAXExPnq5dTZsLyRrR80BOSHWR9vGiSzaEFjqIug8NltYdTc+fBXKhtSCf0qhPzBGRLrj5HQfE5HjcJ5I0QJoAvwBmOg3+c6kqjYsIkOAIQAtWob/KfBfz1zCoZ1bkN5oTw6vpLSUz79ZwMev3B7GlgVfYXEZL3+3umL5hQsPY0v+bg5pksz05dsYN2f9fu95ZuoKoOocbnZBEY0TY8kuKCZKoEGsj3z3q3KjhBj+ekJHRn67ms0B9AQjRUKsj46ZSSzakLtXLzRrx27WbCvYr/7ijc7FwKpyuEUlZcRFR1Hk9nKjfXt6t7HRURzSLIVlm/PYVRw5D33xUg7X03PbquoPQDqQAVzm/uzrBuTN7LlRI6AvP6o6WlX7qWq/xunpwWhyrfxnyhzOGrh3T/a72cvo0CqTZhkNw9Sq0EiI9VVcfR7YKZ3Fm/IoLC7j5407OLJtGinxTl8hMdZHemJgj9ibvXY7x3VsDMCRbdNY5AafhFgfd5/UiffmrGNZVn4QjiY44qoIgNsLi0lPiiPG55y/6CipNhfsL3tnEZkpcQCkJ8WyvcBJU/iihEObp7B62869csfhJoBPJKBXJAh0AvKIJCJdcSZH34YzeiJLVYtF5ASgjVttOvCWiDyBc7xnAa+Eo721UbiriO9mL+PR2y/aq/yTKfMOinTCLce1o1vTZJLjoxl5UQ8+nLehIsB+uXQLLVLjufkP7ShTZf32Xbzi9nbX5+7igznrufeUzs4wlTLljZm/sdXNzVZn6q9bGfqHdjx3fnfyd5fy/NdOj/jUrpk0SY7j/J7NOb9ncwAem7ysIlURLl2aJpPaIIZon3B42zR+yy6o6M1tyt1Fq0YJxPik4sKWqjJ/bS6FRaWs2baTQ1ukIjgXtlZsyd8rN1uVTTt20aVJMn3bpFUMCwNonhpPfIyPVo0SaNUoAYBF63MrUhXh5KFRYYgGmPkWkThVDft3Lb9hYeD8gbtXVf/r5m3/g3PBbB5wDHCaqq72u2i2BlgH/FLTsLCevfvq/6b+EKzD8LybPwz/sKlIN+yYtuFuQkS7+aKTWPbzvDqFyyYdu+tlz3wYUN1nzzlktqr2q8v+6iqQuRSOAF7H6UG2FpGewHWqekuwG1cZVfVVUb4V6F/FukdxxhAbYw4iziN2vNPFDSSx8zxwJs7XdlR1PnZrrzEmQkRJYK9IEEgON0pV1+zzV6Ru9w8aY0w98VAHN6CAu9ZNK6iI+IBbgMDusTTGmCASINpDETeQgHsTTlqhNc5Qqy/dMmOMCTsPxduA5lLIwrlTyxhjIopE0CPQAxHIKIVXqeTGAVUdEpQWGWNMLXgo3gaUUvjS7/d44DycSWCMMSbsImUEQiACSSm8778sIu8AXwStRcYYEyDh4JuAfF/t2HPbrDHGhE8EjbENRCA53Bz25HCjgGxgeDAbZYwxgZKIeWJZzaoNuO6zzHriPMcMoEwDnXzBGGOC7KB6TLobXCeqaqn7smBrjIkoXrq1N5C5FH4UEe/PB2iMOSiJSECvSFBlwBWR8nTDsThBd6mIzBGRuSIyJzTNM8aYqjmPSQ/sVfO25A0RyRKRn/3KRojIehGZ575O91t3j4gsd2PjqYG0t7oc7o84zwM7N5ANGWNMONTjnWZv4TzLccw+5c/uO3+2iHTDuQP3UKA58KWIdFbVaif2qi7gCoCqrqhlo40xJiTq86KZqk4XkbYBVj8HGOc+lGGViCwHjgCqfWpBdQE3Q0SqfFKhqj4TYMOMMSZoatHBTReRWX7Lo1V1dADvGyYiVwCzgL+qag7OQ2pn+NVZ55ZVq7qA6wOSwEOD3IwxvzNCVOAhausBPGJnFPAwzr0IDwP/AK6h8rhY4yiu6gLuRlV9qJaNM8aYkBGCO3mNqm6u2Jczkdcn7uI6oJVf1ZbAhpq2V921O+vZGmMimziPgQ/kdUCbF2nmt3geUD6CYRJwqYjEiUg7oBPOQINqVdfDPfGAWmiMMSFSnz1cEXkPGICT610HPAAMEJFeOOmC1cANAKq6SEQ+AH4BSoChNY1QgGoCrqpm1/UAjDEm2OprWJiqDq6k+PVq6tf6aeAHMluYMcZEjAi5iSwgFnCNMZ4lBDY/QaSwgGuM8S6p1zvNgs4CrjHGs5w7zSzgGmNMSHgn3FrANcZ4nIc6uBZwjTFeFjlz3QbCAq4xxrNslIIxxoSQXTQ7CPiihLTEmHA3I2KNu6q2ky79/qQdPizcTYhou1fWONdLzQRLKRhjTChYSsEYY0LIerjGGBMi3gm3FnCNMR4mgM96uMYYExoeircWcI0xXiaIh5IKFnCNMZ5mPVxjjAkBZ1iYdyKuBVxjjHeJ9XCNMSZk7NZeY4wJAWcC8nC3InAWcI0xnmajFIwxJkQ8lFGwgGuM8Tbr4RpjTAhYDtcYY0JFxEYpGGNMqHgn3FrANcZ4mJNS8E7ItYBrjPE074RbC7jGGK/zUMS1gGuM8TRLKRhjTIh4J9xawDXGeJ2HIq4FXGOMZwl2p5kxxoSGzYdrjDGh46F4awHXGONlgnioixsV7gYYY0xdiAT2qnk78oaIZInIz35ljUTkCxH51f2Z5paLiDwvIstFZIGI9AmkrRZwjTGeJbV4BeAtYNA+ZcOBr1S1E/CVuwxwGtDJfQ0BRgWyAwu4xhhvq6eIq6rTgex9is8B3nZ/fxs41698jDpmAA1FpFlN+7CAa4zxNAnwvwPURFU3Arg/M93yFsBav3rr3LJq2UWzCNXznAdISojDFxVFtC+KKWPu4pp732D5miwAcvMLSU1qwPSxw2vY0sGpx9n/t+f8REcxdczdAIx+fxqvfjCdaF8UJx/bnYduPbeGLUWmFk0aMmrEFWQ2TqFMlbcnfscr46btVSc1uQEv3v8n2rVMZ1dRMbc8PJbFKzbWab+xMdGMevByenVtTXbuTq659w3WbsxmwBFdeWDY2cTGRFNUXML/Pf8R38xaVqd91ZdaXDNLF5FZfsujVXX0ge62kjKt6U0RG3BFpAnwLHAUkAMUAU+p6sQ6bncacIeqzqqpbrhNGnUrjRsmVSy/8dg1Fb/f99wEUpIahKNZEeM/L/95r/PzzaxlfPr1Qr597x7iYmPYkp0XxtbVTUlJGfc9N4EFS9eRlBDH1DF3M23mEpau2lRR569Xn8rCZeu4/K5X6dSmCU/ffTHn3vxCQNtv1awRLz1wOWfd+M+9yi8/pz+5Owrpe/6DnH9yX0bccg7X3vsm27bnM/j2V9i0NZdDOjTjw+eHcugZ99XrMR+Q2o3D3aqq/Wq5h80i0kxVN7opgyy3fB3Qyq9eS2BDTRuLyJSCOOM8PgKmq2p7Ve0LXIpzUL97qspHX87lglP6hrspEeWN8d9w25UnExcbA0BGo+Qwt+jAbd62gwVL1wGQX7CbZas30Syj4V51urRryvSflgLw65rNtG7WqOKYLz7tcL586w6mjx3Os/dcSlSAz6FevumcAAANsklEQVQ57bgevPffmQB8PGUuxx/eBYCFy9axaWsuAItXbCQ+NobYmMjorwU5pTAJuNL9/UrgY7/yK9zRCkcBueWph+pEZMAFBgJFqvpyeYGqrlHVF0QkXkTeFJGFIjJXRE4AqKa8gYiMc4duvA94olsowAW3jOSEK57irYnf7bXuh7kryGyUTIfWmZW/+XdARDh/2IsMuPxJ3prwLQDL12Txw7wVnHTV05wx5DnmLFoT5lbWj1bNGtGjS0tmL1q9V/nPv67nzBN6AdCnWxtaNW1E88yGdG7bhPNO7sOga5/huMueoLSsjIsGHR7QvppnprJ+cw4ApaVl7MgvpFFq4l51zh7YiwXL1lJUXFL3g6sjoV6Hhb0H/AB0EZF1InIt8ARwsoj8CpzsLgN8CqwElgOvAjcH0t7I+BO1v0OBOVWsGwqgqoeJSFdgsoh0rqb8JqBAVXuISI9qthtR/vfa7TTLSGVLdh7nD3uRzm2acHSfjgCMnzyb80/9ffduP3vtLzTLaMiW7DzOG/Yindo2paS0jO15BXzx5h3M+WUNV9/7BvM+GuGpgfH7SmwQy5gnr+OeZ8aTt3PXXuuee/sLHv/rhUwfO5xflm9gwbJ1lJaWcfzhXejZtTVTxtwFQHxcDFuy8wF456nradOiMTHRPlo2bVRxDeDlcdN49z8zKo1M/onJru2bMuKWczh/2MjgHPABqK//u6o6uIpVJ1ZSV3FjTm1EasDdi4iMBI7FyeOuA14AUNUlIrIG6Oyur6z8OOB5t3yBiCyoZj9DcMbU0ap166AdTyCaZaQCztfiMwb0ZPYvazi6T0dKSkr5ZNp8prx9Z1jbF27lX68zGiVz5oAezFm0mhaZDTnrhJ6ICH0PbUuUCNu255Oe5s3UQrQvirefvJ5/fzaLT6bO32993s5dDHvoXxXL8z9+kDUbttG/d0fG/XcmD42ctN97Lr/rVaDqHO6Gzdtp0SSNDVnb8fmiSElqQE7uTgCaZzbknaeGcNMD77B6/db6PNS68dDf00hNKSwCKu7cUNWhOH9lMqj69FZ32mu8eujuZ7Sq9lPVfunpGYG2td7tLNxd0ZvZWbibqTOXcEgHZ4jftJ+W0qlNE1o0SQtb+8Jt3/MzZcYSDunQnNMH9GD6T86V8+VrNlNUXLLXRTWveeH+y1i2ehMvvTul0vUpSQ2IifYBcMW5R/P93OXk7dzF9J+WcvbAXqSnOcfeMCWBVk0D+7x89s1CBp9xJADnDOxdcT5Tkhrw/rM38tDIScxcsLKuh1avotwn99b0igSR2sOdAjwmIjepavkdHAnuz+nAZcAUN2XQGlgaQPlUEekO9AjdYRyYLdl5XH6n0xMpKS3jwlP7cVL/bgBMnDz7d3+xbMu2PP7k9tRKS0q5YFA/Tjq6G0XFJQx7aCz9L3mU2Bgfo0Zc7tl0wlE923PpGUey6Nf1FV/7Hx45iZZNGwHw5oRv6dKuKaNGXE5pWRlLV23ilofHArB01SYeffkTJrw4jCgRiktKufOpD1i7KafG/b7z8fe8/OAVzJ7wADk7dnLt394E4PqLj6NdqwzuvG4Qd17n3Ix1/rAX2ZqTH4zDrxUv/R8WJxURedwhGM8CRwJbgJ3AyzhXCV8G+gIlwO2qOlVE4qsobwC8CXQD5gEdgVtrGhbWp28//W7GT0E5toOBVwNZKKUdPizcTYhou5d+QFlBVp0+SN179tEJk78NqG6XpomzD2BYWL2K1B5u+V0dl1ax+qpK6u+qorywmu0YYzzMJiA3xphQsQnIjTEmdDwUby3gGmO8zFsTkFvANcZ4mofirQVcY4x31WJy8YhgAdcY420eirgWcI0xnmbDwowxJkQsh2uMMaEgEOBUvxHBAq4xxuO8E3Et4BpjPKt8AnKvsIBrjPE0D8VbC7jGGG+zHq4xxoSI3dprjDEh4p1wawHXGONhgT6RN1JYwDXGeJrdaWaMMaHinXhrAdcY420eircWcI0xXhY5j0APhAVcY4xnee1Os6hwN8AYY34vrIdrjPE0L/VwLeAaYzzNhoUZY0wo2I0PxhgTGl67aGYB1xjjaZZSMMaYELEerjHGhIiH4q0FXGOMx3ko4lrANcZ4loCnbu0VVQ13GyKSiGwB1oS7HX7Sga3hbkSEs3NUvUg7P21UNaMuGxCRz3COKxBbVXVQXfZXVxZwPUJEZqlqv3C3I5LZOaqenZ/ws7kUjDEmRCzgGmNMiFjA9Y7R4W6AB9g5qp6dnzCzHK4xxoSI9XCNMSZELOAGkYicJyIqIl3D3RavE5FSEZknIvNFZI6IHF3L948QkTuC1b5wEJEmIvKuiKwUkdki8oOInFcP250mIjaaIQgs4AbXYOBb4NK6bkhEfHVvjqcVqmovVe0J3AM8Hu4GhZOICPARMF1V26tqX5zPWcvwtsxUxwJukIhIEnAMcC1uwBWR90XkdL86b4nIBSLiE5GnReQnEVkgIje46weIyFQReRdY6JZ95PZmFonIEL9tXSsiy9zeyasi8qJbniEi491t/yQix4TuLARNCpADznkWka/cXu9CETmnvJKI/E1ElorIl0CXcDU2SAYCRar6cnmBqq5R1RdEJF5E3nTPx1wROQGgmvIGIjLO/ey9DzQIzyEd/OzW3uA5F/hMVZeJSLaI9AHGAZcAn4pILHAicBNOUM5V1cNFJA74TkQmu9s5Auiuqqvc5WtUNVtEGgA/ich4IA64H+gD5AFTgPlu/X8Cz6rqtyLSGvgcOCTIxx4MDURkHhAPNMMJOAC7gPNUdYeIpAMzRGQSzrm4FOiN8zmfA8wOfbOD5lCcY6rMUABVPcxNZ00Wkc7VlN8EFKhqDxHpUc12TR1ZwA2ewcBz7u/j3OX7gefdoDoI5+tgoYicAvQQkQvd+qlAJ6AI+NEv2ALc6pena+XWawp8rarZACLyb6CzW+ckoJvsud88RUSSVTWvfg836ApVtReAiPQHxohId5zb6R8TkeOAMqAF0AT4AzBRVQvc90wKT7NDQ0RGAsfifGbWAS8AqOoSEVmD83k4tory44Dn3fIFIrIg9Efw+2ABNwhEpDFOD6y7iCjgAxS4C5gGnIrT032v/C3ALar6+T7bGQDs3Gf5JKC/qhaIyDScHl91s3dEufUL63pckUJVf3B7sxnA6e7PvqpaLCKrcc4JOOf8YLUIuKB8QVWHuudkFrC+ivdU9zk5mM9VxLAcbnBcCIxR1Taq2lZVWwGrcHoY44CrcXpg5QH2c+AmEYkBEJHOIpJYyXZTgRw32HYFjnLLfwSOF5E0EYnG7x8iMBkYVr4gIr3q7SjDxD12H7AN55xkucH2BKCNW206cJ6bn0wGzgpPa4NmChAvIjf5lSW4P6cDl4HzWQJaA0sDLO8O9AhB+3+XrIcbHIOBJ/YpGw/8EbgVGANMUtUid91rQFtgjnv1eQtODnhfnwE3ul/5lgIzAFR1vYg8BswENgC/ALnue24FRrrvicb5x3VjPRxjqJXncMHpqV2pqqUiMhb4j4jMAuYBSwBUdY57AWgezqxv34Sj0cGiqioi5wLPishdOJ+ZncDdwMfAyyKyECgBrlLV3SLyUhXlo4A33c/IPJw/4CYI7E6zg4SIJKlqvtvDnQi8oaoTw90uY8wellI4eIxwe4A/46QvPgpze4wx+7AerjHGhIj1cI0xJkQs4BpjTIhYwDXGmBCxgGsOiOyZvetnEfm3iCTU/K4qtzVARD5xfz9bRIZXU7ehiNx8APuodLawqsr3qfOW312AgeyrrYj8XNs2moOfBVxzoMpn7+qOczvpXmN7xVHrz5eqTlLVfccw+2sI1DrgGhMJLOCa+vAN0NHt2S12B9jPAVqJyCnizNM6x+0JJwGIyCARWSIi3wLnl29IRK6SPTOdNRGRieLMgTtfnDlwnwA6uL3rp916d8qemdYe9NtWrWYLE5Hr3e3MF2eGNf9e+0ki8o04M7Kd6davdJY3Y6piAdfUiXujxWm400fiBLYxqtob586n+4CTVLUPzn3+t4tIPPAqzu22f8CZfKcyz+NMytMTZ/avRcBwYIXbu77TnfinE86sar2AviJynIiUzw/bGyegHx7A4UxQ1cPd/S3GmcWtXFvgeOAMnLu14vGb5c3d/vUi0i6A/ZjfKbu11xwo/1ttvwFeB5oDa1R1hlt+FNANZ7pJgFjgB6ArsEpVfwUQkX8BQ9jfQOAKAFUtBXJFJG2fOqe4r7nuchJOAE6m9rOFdReRR3DSFknsmesC4ANVLQN+FZGV7jFUNcvbsgD2ZX6HLOCaA1UxXWI5N6ju9C8CvlDVwfvU60X9zU4lwOOq+so++7jtAPbxFnCuqs4XkauAAX7r9t2WUvUsb21ruV/zO2EpBRNMM4BjRKQjgIgkuLNULQHaiUgHt97gKt7/Fc7k2OX50hScCdaT/ep8DlzjlxtuISKZHNhsYcnARnFmbbtsn3UXiUiU2+b2OJMHBTrLmzGA9XBNEKnqFren+J44k64D3Oc+BWMI8F8R2Yrz3LfulWziz8BoEbkWKAVucufC/c4ddvU/N497CPCD28POB/50gLOF3Y8z49oanJy0f2BfCnyNM7n5jaq6S0QCneXNGMDmUjDGmJCxlIIxxoSIBVxjjAkRC7jGGBMiFnCNMSZELOAaY0yIWMA1xpgQsYBrjDEhYgHXGGNC5P8BZsoL9q/m05sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "\n",
    "tuned_parameters = [{'var_smoothing': [1e-6, 1e-07, 1e-08, 1e-09, 1e-10, 1e-11, 1e-12, 1e-13, 1e-14, 1e-15]}]\n",
    "\n",
    "clf = GridSearchCV(GaussianNB(), tuned_parameters, scoring='f1_macro')\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "#Evaluation\n",
    "\n",
    "print(\"Best parameters set found on development set:\\n\", clf.best_params_)\n",
    "print(\"\\n Best Score\", clf.best_score_) \n",
    "\n",
    "print(\"\\n Grid scores on development set:\")\n",
    "means = clf.cv_results_['mean_test_score']\n",
    "stds = clf.cv_results_['std_test_score']\n",
    "for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "    print(\"%0.3f (+/-%0.03f) for %r\" % (mean, std, params))\n",
    "\n",
    "print(\"Detailed classification report:\")\n",
    "print(\"The scores are computed on the full evaluation set. \\n\")\n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(classification_report(y_true, y_pred))\n",
    "\n",
    "#Lets print confision matrix\n",
    "labels =['Average','Bad','Good']\n",
    "disp = plot_confusion_matrix(clf, X_test, y_test,\n",
    "                                 display_labels=labels,\n",
    "                                 cmap=plt.cm.Blues)\n",
    "\n",
    "print ('\\nTesting score: ',clf.score (X_test, y_test)) # print the testing score(accuracy)\n",
    "print ('\\nTraining Score: ',clf.score (X_train, y_train)) # print the trainig score(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['fit_time', 'score_time', 'test_f1_macro']\n",
      "test_f1_macro [0.47942543 0.46982902 0.47266061 0.46363159 0.47202167]\n"
     ]
    }
   ],
   "source": [
    "#Cross Validate\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.metrics import recall_score\n",
    "\n",
    "scoring = ['f1_macro']\n",
    "scores = cross_validate(clf, X, Y, cv=5, scoring=scoring)\n",
    "print(sorted(scores.keys()))\n",
    "print('test_f1_macro', scores['test_f1_macro'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K select Percentile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.000210523684764536 0.005160091115494809 0.1361031518624642 0 0]\n",
      " [0.06578947122576177 0.013897588526412541 0.14469914040114612 0 0]\n",
      " [0.005263155277008304 0.010694145453587434 0.11891117478510028 0 1]\n",
      " ...\n",
      " [0.05263157645429362 0.859810398981424 0.1346704871060172 0 0]\n",
      " [0.000988368423653601 0.005792679656356735 0.1017191977077364 1 0]\n",
      " [0.17105262939750693 0.057828596847734885 0.13753581661891118 0 0]] ['Bad' 'Bad' 'Bad' ... 'Average' 'Bad' 'Average']\n",
      "4800 4800 1200 1200\n"
     ]
    }
   ],
   "source": [
    "#Load Kbest data\n",
    "arr2= np.load('data_select_percentile.npy', allow_pickle= True)\n",
    "X= arr2[:,0:-1]\n",
    "Y= arr2[:,-1]\n",
    "\n",
    "\n",
    "### Split arrays or matrices into random train and test subsets - 20% for Test, 80% to train\n",
    "X_train, X_test, y_train, y_test= train_test_split(X, Y, test_size=0.20,  random_state = 200)\n",
    "\n",
    "print(X_train,y_train)\n",
    "print(len(X_train),len(y_train),len(X_test),len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters set found on development set:\n",
      " {'var_smoothing': 1e-06}\n",
      "\n",
      " Best Score 0.5471450832004444\n",
      "\n",
      " Grid scores on development set:\n",
      "0.547 (+/-0.012) for {'var_smoothing': 1e-06}\n",
      "0.547 (+/-0.012) for {'var_smoothing': 1e-07}\n",
      "0.547 (+/-0.012) for {'var_smoothing': 1e-08}\n",
      "0.547 (+/-0.012) for {'var_smoothing': 1e-09}\n",
      "0.547 (+/-0.012) for {'var_smoothing': 1e-10}\n",
      "0.547 (+/-0.012) for {'var_smoothing': 1e-11}\n",
      "0.547 (+/-0.012) for {'var_smoothing': 1e-12}\n",
      "0.547 (+/-0.012) for {'var_smoothing': 1e-13}\n",
      "0.547 (+/-0.012) for {'var_smoothing': 1e-14}\n",
      "0.547 (+/-0.012) for {'var_smoothing': 1e-15}\n",
      "Detailed classification report:\n",
      "The scores are computed on the full evaluation set. \n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Average       0.45      0.41      0.43       405\n",
      "         Bad       0.54      0.81      0.65       392\n",
      "        Good       0.69      0.41      0.51       403\n",
      "\n",
      "    accuracy                           0.54      1200\n",
      "   macro avg       0.56      0.54      0.53      1200\n",
      "weighted avg       0.56      0.54      0.53      1200\n",
      "\n",
      "\n",
      "Testing score:  0.5304867644125714\n",
      "\n",
      "Training Score:  0.546317183561836\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEKCAYAAABewe3GAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl8FdX9//HXOwmQQBaWQEAWQQEBARFxwa24rxVtS4u71opal6q1ru1P7aJ+tRVrtVqsirZVxLZUXCoqiohFVpFFUDbZZF/CkrAkfH5/zAQDZLkhucvg5/l43EfuPXPumTOX8Mm5Z858RmaGc865+EtLdgecc+7bwgOuc84liAdc55xLEA+4zjmXIB5wnXMuQTzgOudcgnjAdc65BPGA65xzCeIB1znnEiQj2R1IVVm5TSynRetkdyNl7dzpVyhWp2VOg2R3IaUtW7KY9evWqDZtpOceaFZSHFNdK149yszOrM3+assDbiVyWrRmwMPDk92NlFW0rSTZXUh5d/Q7ONldSGk/OPOEWrdhJcU0OOSHMdXdOu3J/FrvsJY84DrnIkyg6MyMesB1zkWXgLT0ZPciZh5wnXPRplpNAyeUB1znXIT5lIJzziWOj3Cdcy4BhI9wnXMuMeQjXOecS5gIrVKIzljcOef2Ep40i+VRXUtSpqSJkj6TNEvS/WF5B0kTJM2V9Iqk+mF5g/D1vHB7++r24QHXORddIphSiOVRvW3AyWZ2GNALOFPSMcD/AYPNrBOwHrgqrH8VsN7MOgKDw3pV8oDrnIu2OhrhWmBz+LJe+DDgZOCfYfkLwPnh8/7ha8Ltp0hVR3YPuM65CKu7KQUASemSpgGrgHeB+cAGMytLHrIUKMtq1RpYAhBuLwSaVdW+nzRzzkWXgPSYT5rlS5pc7vUQMxtSvoKZlQK9JDUGRgBdK2inLFVeRaPZKtPoecB1zkVb7MvC1phZn1gqmtkGSWOAY4DGkjLCUWwb4Ouw2lKgLbBUUgaQB6yrql2fUnDORVidrlJoHo5skZQFnArMBj4AfhBWuxx4LXw+MnxNuP19M/MRrnNuP1Z3Fz60Al6QlE4wGB1uZm9I+hwYJum3wKfAs2H9Z4G/SZpHMLIdWN0OPOA656Ktji7tNbPpwOEVlC8AjqqgfCswoCb78IDrnIuu2NfYpgQPuM65aIvQpb0ecJ1zEeb5cJ1zLnF8SsE55xLA8+E651yi+JSCc84ljp80c865BPE5XOecSwD5lIJzziWOj3Cdcy4xqsn5nVI84DrnIiu4w44HXOeciz8JpXnA/db7Ua9WdC3IYfO2En4/ZsFe2/sd3IzebfIASBMU5DTg/739BcU7du7zPtPTxEWHH0Cbxlls2V7K3yYvZX3xDjo3b8TZXVuQkSZKdhpvfL6SeWuK9nk/deHSPm3o0SqXTdtK+M07X+61PTMjjSuPbkfThvVIk3jvy9WM/2p9rfbZsF46P+nbjmYN67O2aDt/Hb+Yoh2lHNmuMacf0hyAbSU7eXnqMpYVbq3VvpJt0+Zi7v/jP5m/aCUS3HvzAN7/30zGTphNvYx02rRqxv23DCAnOyvZXa21KI1w43p6T9IFkkxSl3juJxVNWlzIM58srnT7mPlrefTDBTz64QLemr2K+WuKYg62TbLqcd2xB+5VfnS7xhTtKOXB0fMYO38t53ZrAcCWbaU8N2EJvx+zgGGffs1Fh7fe672JNv6r9fzpo4WVbu/XsRnLN27ld+/OZfCY+Xz/sFakx/gfq1PzRlx2ZJu9ys/o0pw5Kzdz79tfMGflZk7vEgTZtVu2M3jMAn737lz+O3sVFx+x93uj5uG/jOTYIw5hxJDbeOWJmzmobQuOObwTrz51C8P/fAsHts7nueEfJLubdUJSTI9UEO/1FBcC44ghMW91wqTAkbFgXRFF20tjqnt46zw+XVa463XvNnn87IQO3Pqdg/hBz1YV3jipIt1b5jB5SdDO9OUb6ZTfCIBlG7eycVtwD7wVm7aRkS7Sk/w1bN6aLWzZXlLpdiMY5QI0yEhjy/ZSdobJ9E/r3Jw7TunIPad14txuBTHv87DWeXyyKBglf7JoPb1aB98wFqwtomhH8G+1cG0RTRrW25dDShmbi7YydeZCLjjjSADq1csgJzuLvr07kxHe/6tHl3asXFNYVTOR4QEXkJQNHEdw7/aBYdkrks4uV2eopO+Hd8p8RNIkSdMlXRNu7yfpA0kvATPCsv9ImiJplqRB5dq6StKXksZIekbSE2F5c0n/CtueJOm4eB3zvqiXLrq0yGb68o0AtMiuT68DcvnTuIU8+uECdprtmnqoTm5mBhuKdwCw06C4ZCeN6u/+d6pnqxyWFW6ldGeVdwJJujHz1tIyN5OHzu3KL8/ozKuffo0BXQuyaZ5Tn/8bPY8H3p1LuyZZdAz/sFQnp0EGG7cGQX7j1hJyGuz9N/zYDk2YtXxTXR5Kwi1bvo4meY24d/CrDLzhj9z/2D8p3rp9tzqvvTOZ4/ockqQe1iHV4JEC4jmHez7wtpl9KWmdpN7AMOBHwFuS6gOnANcRBOVCMztSUgPgY0nvhO0cBXQ3s7Lvnz82s3XhPYcmSfoX0AD4FdAb2AS8D3wW1v8jMNjMxklqB4yi4jtxJsWhBTksXPfNdEKn/Ea0aZzJzSceBAQBeXM4Ur7iyDY0bVif9DTRJKset34nqPPRgrVMWlJY8S1Ey8XVgpwGnNOtgCHjF8X1mOpCt4Iclm4o5rEPF9C8UX1u+s5BzHvnS7oW5NCtIIe7T+sEBKPfFtn1mbdmC7ef3JGMdNEgI41G9dN31RkxfTmzV26udp+dmzfi2A5N+cMH8+N6bPFWUrqTOfO+5o5r+9OjSzsefnokzw3/gOsvOwOAvw57n/T0NM4+aa+bG0SOSJ3RayziGXAvBB4Lnw8LX/8KeDwMqmcCY82sWNLpQE9JZTdqywM6AduBieWCLcBNki4In7cN67UEPjSzdQCSXgU6h3VOBbqV+0fJlZRjZnsNY8IR8yCA7PxWtTr4WPVqnbvbdIIEk5cU8tbsVXvVHTppKRDM4Q48/ACe+t/ugbNwawmNs+pRuLWENEFWRtqur8p5mRlceWQbXv50GWuLdsTxiOpG3w5NeGdO8Bms3rKdtVu2U5DTAAFvz1nFuAV73xz14ffnAcEcbt/2TXgx/LzKbNpWQm5mMMrNzcxg07Zvpnxa52VySZ82PPHRQrbEOBWUqgry82iRn0ePLu0AOPX4Hjz/6hgARr43hbETZ/OXB66OVKCqSlpadK40i0tPJTUDTgb+Kukr4BcEI9ttwBjgjPD1sLK3ADeaWa/w0cHMyka4W8q1248ggPY1s8MIbuiWSdVfGNLC+mVtt64o2AKY2RAz62NmfbLymu7LoddIZkYaBzdrxKwV33Rn7uot9GyVQ3Y4FZBVL40mWbHNKc5asYk+bYPph56tcpm7Zsuu/fzk6Ha8OXsVX60rruOjiI/1RTs4pEUOEEwFFOQ0YM2W7Xy+chPHtm9Kg/TgVzcvM6PCqYGKTP96I8cc2ASAYw5swmfhH7omWfUYdOyBDJ24hFWbt1fVRCTkN82hZfM8vlq6GoCJ0+ZxULsWfDz5C4a+OobH7r2crMz6Se5l3YnSHG68Rrg/AF40s2vKCiR9CBxPEGR/AvQBrgg3jwKuk/S+me2Q1BlYVkG7ecB6MysKVz4cE5ZPBAZLakIwpfB9wjlf4B3gBuCRsB+9zGxanR1pJS7p3ZqD8xvSqH4GvzqtE6O+WL3rLPv48MRNj1Y5fLF6M9tLv/nev3Lzdt6es5pBfQ9EgtKdxr9nrGB9cfWj0gmLN3BR79bcdUpHiraX8rcpwQjv+A5NadaoPqd1bs5pnYMz80PGL9o1VZEMPz66HZ2bNyK7QQYPnNOFN2at3HUi76MF63jr85VcdmRbfnl6J4QYMX05W7aXMnvlZlrmrOcXpxwMBMu4np+wZLfRamVGzVnFT45px3EdmrKuaDvPjA9WkZzTrYDs+ukM7B2s3ti503ho9Lw4HXli3HFtf+5++GVKSkpp3bIp998ygEtufoLtO0q47p6/AtDjkHb88sbvJbmntZRC87OxUDW3Ud+3RqUxwENm9na5spsI5k5vAlYAI83synBbGvBb4LsEH99qgjngw4HbzOzcsF4D4D9Aa+ALoDlwn5mNCacDbgO+JriX/Dozu0dSPvBkuO8MgmmMa6s7hhYdu9uAh4fX9qPYbxVtq3yFgQvc0e/gZHchpf3gzBOY+dnUWoXLjPyDrPG5D8RUd+0LF04xsz612V9txWWEa2b9Kih7vNzLZnts2wncHT7KGxM+yuptA86qZLcvmdkQSRnACIKRLWa2hmD6wjm3n/GTZslzn6RTCeZ03yEYCTvn9nN+aW8SmNltye6Dcy7BFK1Le/ebgOuc+3bygOuccwkSpYAbnRXDzjm3h7KTZnWxDldS2zCVwOwwdcDPwvL7JC2TNC18lE9PcJekeZK+kHRGdfvwEa5zLtrqboBbAvzczKZKygGmSHo33DbYzH6/226lbgR5Yg4FDgDek9TZzCpdFO4B1zkXXaq7S3vNbDmwPHy+SdJsgjX/lekPDAuXqy6UNI8g98v4yt7gUwrOuUirwZRCvqTJ5R6DqmizPcGFVxPCohvCTIbPhVe0QhCMl5R721KqDtAecJ1zERd7esY1ZblSwseQCpsLUsv+C7jZzDYCTwEHA70IRsB/KLfnPVV56a5PKTjnIq0uVylIqkcQbP9hZv8GMLOV5bY/A7wRvlxKkLGwTBuC1AKV8hGucy6yYp1OiHGVgoBngdlm9mi58vK5Wi8AZobPRwIDJTWQ1IEgVezEqvbhI1znXKTV4Qj3OOBSYIaksoyCdwMXSupFMF3wFXANgJnNkjQc+JxghcP1Va1QAA+4zrmIq6tcCmY2jornZd+q4j2/A34X6z484DrnIi1KV5p5wHXORZcnr3HOucQQwX0Ao8IDrnMuwjwBuXPOJUyaJyB3zrkEkE8pOOdcQggf4TrnXML4CNc55xLET5o551wi+Byuc84lhlCdJSBPBA+4zrlI8xGuc84liM/hOudcIvgcrnPOJUaQSyE6EdcDrnMu0iIUbz3gOueiza80c865RPB8uPuHFo3qc9Ox7ZPdjZTV66zbk92FlPfUgCeS3YWU1iCj9utnPR+uc84ljOfDdc65hIlQvPWA65yLMPlJM+ecSwhfh+uccwnkAdc55xIkQvHWA65zLtqiNMKNTiJJ55zbU5i8JpZHtU1JbSV9IGm2pFmSfhaWN5X0rqS54c8mYbkkPS5pnqTpknpXtw8PuM65yAoSkMf2iEEJ8HMz6wocA1wvqRtwJzDazDoBo8PXAGcBncLHIOCp6nbgAdc5F2lpUkyP6pjZcjObGj7fBMwGWgP9gRfCai8A54fP+wMvWuAToLGkVlX2dd8O0TnnUkNdTSns3qbaA4cDE4ACM1sOQVAGWoTVWgNLyr1taVhWqUpPmknKreqNZraxuk4751w8qWbJa/IlTS73eoiZDdm7TWUD/wJuNrONVbRf0QarqgNVrVKYFb65fKNlrw1oV1XDzjmXCDW40GyNmfWpqoKkegTB9h9m9u+weKWkVma2PJwyWBWWLwXalnt7G+DrqtqvNOCaWdvKtjnnXKqoq0t7FQxlnwVmm9mj5TaNBC4HHgp/vlau/AZJw4CjgcKyqYfKxLQOV9JA4CAze0BSG4I5jSk1OhrnnKtjIlipUEeOAy4FZkiaFpbdTRBoh0u6ClgMDAi3vQWcDcwDioArq9tBtQFX0hNAPeBE4IGw4aeBI2tyJM45Fw91lbvGzMZR8bwswCkV1Dfg+prsI5YR7rFm1lvSp+FO1kmqX5OdOOdcXGj/y4e7Q1Ia4dk3Sc2AnXHtlXPOxShC8TamgPskwVm75pLuB34I3B/XXjnnXAwEMV3UkCqqDbhm9qKkKcCpYdEAM5sZ324551xs9scE5OnADoJpBb86zTmXEvblKrJkqjZ4SroHeBk4gGBh70uS7op3x5xzLhZ1lUshEWIZ4V4CHGFmRQCSfgdMAR6MZ8eccy4WqRFKYxNLwF20R70MYEF8uuOcczWzXywLkzSYYM62CJglaVT4+nRgXGK655xzlQtWKSS7F7GraoRbthJhFvBmufJP4tcd55yrAcWcXDwlVJW85tlEdsQ55/bFfjGlUEbSwcDvgG5AZlm5mXWOY7+cc65aUZtSiGVN7VDgeYJjOwsYDgyLY5+ccy5mCvMpVPdIBbEE3IZmNgrAzOab2S+Bk+LbLeeci41ifKSCWJaFbQsT886XdC2wjG/u6ePiZOPmYn792D+Zt2gFkrjvlgG0b9Oc2x/8B1+vXMcBBU155K6Lyc1pmOyu7pMG9TN4c8jNNKiXQXpGOiNHf8pDQ97arc5PLzqZS/v3pbR0J2s2bObGX/+dJSvW12q/jXMb8twDP6Zdq6YsXr6OK+96lsJNxQw4sw8/u+w0ALYUb+PnD73CzLnLarWvZLrh139n1LiZ5DfJYfwr9wDwu6fe4K2x00mTaN40hyfvvYRWzRsnuae1I0F6hOYUYhnh3gJkAzcRJOi9GvhxPDtVFUmlkqZJ+kzSVEnH1vD990m6LV79qysPPz2SY/t05j/P/ILhT95Mh7YteG74BxzdqyOvP3sHR/fqyHPDxyS7m/ts2/YS+l/3OCdc/BAnXvQgp/TtRp/u7XerM/2LJZx82cMcf9GDjBz9KffddH7FjVXguN6dePLeS/Yqv+Xy0xg76Qv6fP/XjJ30BbdcfjoAi75eyznXPMbxFz3II8++zeC7L6zV8SXbhecewz8f3z1V642XnsLHL9/NRy/dxRnHd+fhv/43Sb2rW/vVlIKZTTCzTWa22MwuNbPzzOzjRHSuEsVm1svMDgPuYj+84m3zlq1MnbmAC844CoB69TLIzc5izPhZfPfUIwD47qlH8MH4aOcQ2lK8HYB6GenUy0gnyOf8jXFT5lK8bQcAk2Z8ResW34zGbrzkFEa/8AvGvXQXdw46O+Z9nvWdnrz8xgQAXn5jAmf36wnAxOkLKdxUHO5rIQe0iPbI77jeHWmSu/u3n9zsrF3PtxRvS5kgVFvxuGtvvFR14cMIqrgDpZl9Ly49qplcYD3sutPma0ATgjtU/NLMXgu33QNcRnBL49UElyanrKUr1tEkL5v/9+hwvlywnG6dWnP7tf1Zu2EzzZsGN1Nu3jSXdYVbktzT2klLE2P+dgcd2jTn2VfHMmXWokrrXtq/L+/+73MATjq6Cwe1a8Eplz+CJF7+wzUce/jB/O/T+dXus0XTHFauDW44vXLtRpo3yalgX8fyXriv/c1v/jySYW9OJDc7i9efvinZ3ak1kTp5EmJR1RzuEwnrRc1khfcbygRaASeH5VuBC8LbGucDn0gaCfQGBhLcYz4DmEqKB9zS0lLmzFvGndf1p0eXdvzf06/x3PAPkt2tOrdzp3HixQ+Rm53F3x+5mq4Ht2L2/L3vwffDs46kV9d2nHvNHwE46ZiunHx0F8b+404AGmU14KC2Lfjfp/N59/nbaFA/g0ZZDWiS23BXnfv+9BrvfzK72j4df0QnLjmvL2ddPbgOjzR1/Oqn5/Grn57Ho8+P4pnhY7nrmnOS3aXaSaHRayyquvBhdCI7UgPFZtYLQFJf4EVJ3QlORD4g6USCO1K0BgqAE4AR5ZLvjKysYUmDgEEAB7RO3k2LC/Ib0yI/jx5dgjvRn3Z8T54b/gHNGmezet1GmjfNZfW6jTTNa5S0PtaljZuLGTdlLqf07bZXwP3OUYdw65VncO41j7F9RwkQ/AcbPPQdho7Ye2brtCt/DwRzuBd992iuv//vu21ftW4TBc1yWbl2IwXNclm9ftOubYd2PIDHf3kRA372FOsj/u2hOj8480h+dPNT0Q+4ROvCh0jntjWz8UA+0By4OPx5RBiQV/LNhRqVTo3s0d4QM+tjZn2aNMuPR5djkt80h5bN8/hq6SoAJkyby0HtWvCdY7rx+nvB4Pz196bQr++hSetjbTVrnL1rTjGzQT36HXUIc79auVudHp3bMPiugVz087+wZv3mXeXvj5/Nxef1pVFWcGu9Vs3zyG+SHdN+3x47gwvPPRqAC889mv9+OB2ANgVNePHhq7n23heZv3hVrY8vFZU/rrfHTqdz+4Ik9qZuCEiXYnqkglgTkKckSV0IkqOvBfKAVWa2Q9JJwIFhtbHAUEkPERzvd4G/JKO/NXHHdedz98Mvs2NHKa1bNePXtwxgpxm3P/APRoyaSKvmTXjknr3PwkdFy/xc/nzfpaSnpZGWJka8N5VR42Zy1zXnMG32Yv47dga//tn5NMpqwNCHrgJg6Yr1XPTzv/DBhDl07tCSd54LFptsLtrGNf/vhd2CcmUGv/Auzz/4Yy45ry9LV67nijuDK9h/8ZOzaJrXiN/f8SMASkp2cvLlD8fp6OPvqnue5+Mpc1m7YTOHnvNL7hx0Nu9+PIu5i1aRlibatmzKo3cNTHY360SEVoWhPc8MV1pRamBm2+Lcn1j6UQrMKHsJ3G1mb4bztq8TnDCbRrCE7Swz+6rcSbNFwFLgczP7fVX76X5Yb/vXKE+KVpleZ92e7C6kvPWTUvU0SGo47ug+TJkyuVbhsqBjd7v40X/GVHdw/65TzKxPbfZXW7HkUjgKeJZgBNlO0mHAT8zsxnh3riJmll5J+RqgbyXbfkeQD8I5tx8JlnxFZ4gbyxzu48C5BF/bMbPP8Et7nXMpIk2xPVJBLHO4aWa2aI+/IqVx6o9zztVIhAa4MQXcJeG0gklKB24Evoxvt5xzrnoCMiIUcWOZUrgOuBVoR7DU6piwzDnnkq6uLu2V9JykVZJmliu7T9KyMH/LNElnl9t2l6R5kr6QdEYsfa12hGtmqwiu1HLOuZSiur0F+lCCK2xf3KN88J6rmiR1I4iLhwIHAO9J6mxmVU63xrJK4RkquHDAzAZV917nnIu3uoq3ZjZWUvsYq/cHhoVLZRdKmgccBYyv6k2xTCm8B4wOHx8T5MJN+npc55yDGq1SyJc0udwj1kHjDZKmh1MOTcKy1gTJsMosDcuqFMuUwivlX0v6G/BujB11zrm4ETVKQL5mHy58eAr4DcG3/N8AfyDIB17RTqu9imxfLu3twDeXzTrnXPLEeY2tme1K8BFOr74RvlwKlM9w1Qb4urr2YpnDXc83kTsNWAfcGWN/nXMurhTHO5ZJamVmZSnsLgDKVjCMBF6S9CjBSbNOwMTq2qsy4Ib3MjuM4D5mADst1uQLzjkXZ3V5m3RJLwP9COZ6lwL3Av0k9SIYdH4FXANgZrMkDQc+B0qA66tboQDVBFwzM0kjzOyI2hyIc87FS10FXDOr6EZ2z1ZRv8Y5WmJZpTBRUu+aNOqcc4kSpZtIVnVPswwzKwGOB66WNB/YQjCKNzPzIOycS6rgNunJ7kXsqppSmEhwP7DY703tnHMJtr/cRFIAZlb9rVCdcy4J6vKkWSJUFXCbS7q1so1m9mgc+uOcczUSoQFulQE3Hcim4isqnHMuBYi0CIWoqgLucjP7dcJ64pxzNST2nxFuhA7DOfetJMiI0CRuVQH3lIT1wjnn9sF+M8I1s3WJ7Ihzzu2L/WVZmHPOpbwIxVsPuM656BKx5SdIFR5wnXPRJZ9ScM65hAiuNPOA65xzCRGdcOsB1zkXcREa4HrAdc5FWerkuo2FB1znXGT5KgXnnEsgP2m2H1hfvINXZ1Z71+NvrfeG/ybZXUh5t78xO9ldSGlLC7fWvhHhUwrOOZcIPqXgnHMJ5CNc55xLkOiEWw+4zrkIE5DuI1znnEuMCMVbD7jOuSgTitCkQpRO8Dnn3F6k2B7Vt6PnJK2SNLNcWVNJ70qaG/5sEpZL0uOS5kmaLql3LH31gOuci6xgWZhiesRgKHDmHmV3AqPNrBMwOnwNcBbQKXwMAp6KZQcecJ1z0RXj6DaWEa6ZjQX2vLVYf+CF8PkLwPnlyl+0wCdAY0mtqtuHz+E65yItzpf2FpjZcgAzWy6pRVjeGlhSrt7SsGx5VY15wHXORVaQgDzm6vmSJpd7PcTMhtRi13uy6t7kAdc5F2k1WKWwxsz61LD5lZJahaPbVsCqsHwp0LZcvTZAtclXfA7XORdpdTWHW4mRwOXh88uB18qVXxauVjgGKCybeqiKj3Cdc5FWV+twJb0M9COYelgK3As8BAyXdBWwGBgQVn8LOBuYBxQBV8ayDw+4zrnIquEcbpXM7MJKNp1SQV0Drq/pPjzgOueiS/IE5M45lyjRCbcecJ1zERZMKUQn5HrAdc5FWnTCrQdc51zURSjiesB1zkWaTyk451yCRCfcesB1zkVdhCKuB1znXGSJurvSLBE84Drnoqt2eRISzgOucy7SIhRvPeA656JMKEJDXA+4zrlIi1C89YDrnIsu4VMKzjmXOBGKuB5wnXOR5svCHK8Pf4d5ny+kUXZDBt126V7bF81fwqtDXyevSS4AXXp05ITTjqnVPktKShg5bBQrlq4iq2EmF1xyNo2b5rHgy0V88NbHlJaWkp6ezinnnkD7jm2rbzCF/fut8bw1ejJmxtmn9OH75xzLh+Nn8uKr77N42RqeeOAaDjm4dbK7WSsDeraia0E2m7eV8OjYhRXWOahZQ87rVkBamijaXsLT4xfXap/paWJgrwNonZdJ0fZS/jF1GeuLd9ApvxFndWlOepoo3Wm8OXsV89cW1WpfdSVKc7gpe08zSQWSXpK0QNIUSeMlXVAH7Y6RVNMbydXYYX26MfAnVXe3bYfWXH3rJVx96yU1CrYb1hXyt6de3at82sRZZGZl8tM7r+SoE3vz/lvjAGjYKIsfXnkeg35+Kd8deDqvvfx2zQ4mxSxcvJK3Rk/miQeuYcgj1/PJ1C9Yunwt7du24L7bLqRH1wOT3cU6MXnpBp6dsKTS7ZkZaVzQvSVDJy3h0Q8X8Lcpy2Juu0lWPa7p226v8qPaNqZ4RykPfzCfjxau4+yuwV3Bt2wvYeikpQweu5BXPlvOwMMPqPkBxUOM9zNLlaCckiNcBes8/gO8YGYXhWUHAucltWM10O6gNmxYV7hP750xZTaTPp5GaUkprduuxsq8AAAPH0lEQVS15MzvnUxaWvV/G+fOmr8rcHft0YlRIz7AzGjZusWuOs0LmlFaUkpJSQkZGSn5z1+txctW07VTWzIb1AfgsK7t+Xji5/yo/wlJ7lndWriumCZZ9SrdfnjrPGau2MSGrSUAbNleWm5bLsd1aEqGxOINxYyYsaL6e3gD3QqyeffLNQDMWL6R87sXAPD1xm276qzctI2MNO0a7SZblKYUUnWEezKw3cyeLisws0Vm9idJmZKelzRD0qeSTgKoojxL0jBJ0yW9AmQl55D2tmzRcp559O+8/NcRrF6xFoA1K9fx+Wdfcvn1P+TqWy9BaWnMnDonpvY2FW4ht3EOAGnpaTTIbEBx0dbd6syZMY+CA5pHNtgCtG/bgumzv6JwUxFbt21nwqdzWbV23/64RVl+o/pk1Uvjmr7tuOn49vRunQdAi+z6HHZALn/++Cse+2ghOy0IzrHIy8ygcOsOAHYabN2xk4b10ner06NVDl8XbkuRYOsj3LpwKDC1km3XA5hZD0ldgHckda6i/DqgyMx6SupZRbsJ1bJ1C264+8fUb1CfebMX8uoLr/PTO65g4bzFrFi2iuf++DIAJSWlNMoO/ka8OvR1NqwrZGfpTgo3bOKZR/8OwFEnHM5hRx6KVTOGWb1iLe+/OY6Lrq71zExSHdimBQP7n8Advx1KVmZ9Dj6wJekxfAPY36SlQeu8LIZ8soh6aWnccHx7Fm8opmN+I9rkZXLT8R0AqJcutmwPRsGX9WlD06x6pKeJxln1uPmEoM64heuYvLSw2shUkF2fs7u04JkJtZsrrkspEktjkqoBdzeSngSOB7YDS4E/AZjZHEmLgM7h9orKTwQeD8unS5pexX4GAYMA8lrEd46qQWaDXc87du3A2yPep2hLMRj0PKIrJ519/F7vGXDFd4FgDvf1V97h0usG7LY9Ny+bjRs2kds4h52lO9m2dRtZDTMB2LhhE/984XXOG3gGTfIbx/HIEuOsk4/grJOPAODZl94lv1luknuUeIXFJRRt38yOUmNHaSkL1hbRKjf4vZq8tJC356ze6z0vTl4KBHO4P+zVir/scZKtsHgHeZn1KNxaQpogs14aRTuCqYq8zAwu69OGYdO+Zl3RjjgfXQ1EKOKm6rBgFtC77IWZXU9wq+LmVP7xVvWxx/Tdx8yGmFkfM+vTKK9prH3dJ5s3biG40zIsW7wCM8hqmEn7Tm2ZPWMeWzYHZ4CLi7ZSuH5jTG126nYw06fMBmD2jLm079gWSWwt3sorz73GSWcdR9sOKXKyo5bWF24GYOWaDYyb+DknH9czyT1KvM9XbqJ904akCeqliXaNM1m1eTvz1myhZ6tcGtUPpgKy6qXROCu2sdXnKzfTp20w/dCjVS7z1gS/h5kZaVx5VFv+O2c1i9YXx+eA9lFaeOfe6h6pIFVHuO8DD0i6zsyeCssahj/HAhcD74dTBu2AL2Io/0BSdyAh/zNH/OMtFs1fSvGWrTz+279y4unHUFq6E4Aj+vZk9oy5TB0/nbS0NDLqZXDBxWchieYFzeh3Rl9eGvJvsGAu9swLTtq1fKwqvY46lNeGjeLPDz1PZsNMLrj4bAAmf/wZ69ds4KP3JvDRexMAuGjQ92iU3bCq5lLa/X8YxsZNRWRkpHHjVeeSk53FuImf88Rzb1K4cQv3PPQ3Dm7fiv+75/Jkd3WfXXT4ARzUrBGN6qdz9ykdeffL1aSHgeOTxRtYtXk7X67azC0nHoSZMXHJBlZuCk5ujZqziquPbocEpTuN/8xcwYbikmr3OWnJBgb2OoDbTzqYoh2lvDQ1WPlwbPsm5Desz6md8jm1Uz4Az0xYvNuJumRJjVAaG5WNslKNpFbAYOBoYDWwBXgaeC38eQRQAtxqZh9IyqykPAt4HugGTAM6AjeZ2eSq9t+6cw+77s8j4nJs+4OTDmyW7C6kvFdmrUh2F1LaK7cNYOW8mbWKl90P623/fmdcTHUPadloipnFfUloVVJ1hIuZLQcGVrL5igrqb62kvLiKdpxzEeYJyJ1zLlFSaMlXLDzgOucirS7jraSvgE1AKVBiZn0kNQVeAdoDXwE/NLP1+9J+qq5ScM65GAQJyGN51MBJZtar3HzvncBoM+sEjA5f7xMPuM65SEvAlWb9gRfC5y8A5+9rQx5wnXORpRo8YmQEV6lOCS+EAigIT+KXncxvUem7q+FzuM65aIs9muZLKr8cdIiZDdmjznFm9rWkFsC7kmJLZBIjD7jOuUirwbKwNdWtwzWzr8OfqySNAI4CVkpqZWbLw+sDVu1rX31KwTkXaXU1hyupkaScsufA6cBMYCRQdsni5QQXX+0TH+E656JLkFZ368IKgBHhioYM4CUze1vSJGC4pKuAxcCAKtqokgdc51zE1U3ENbMFwGEVlK8lSJ5Vax5wnXORVZaAPCo84DrnIi1C8dYDrnMu2nyE65xzCVLDy3aTygOucy7SohNuPeA65yIsle7IGwsPuM65SPME5M45lyjRibcecJ1z0RaheOsB1zkXZalzC/RYeMB1zkVW1K4082xhzjmXID7Cdc5FWpRGuB5wnXOR5svCnHMuEfzCB+ecS4yonTTzgOucizSfUnDOuQTxEa5zziVIhOKtB1znXMRFKOJ6wHXORZYgUpf2ysyS3YeUJGk1sCjZ/SgnH1iT7E6kOP+MqpZqn8+BZta8Ng1IepvguGKxxszOrM3+assDbkRImmxmfZLdj1Tmn1HV/PNJPs+l4JxzCeIB1znnEsQDbnQMSXYHIsA/o6r555NkPofrnHMJ4iNc55xLEA+4cSTpAkkmqUuy+xJ1kkolTZP0maSpko6t4fvvk3RbvPqXDJIKJL0kaYGkKZLGS7qgDtodI8lXM8SBB9z4uhAYBwysbUOS0mvfnUgrNrNeZnYYcBfwYLI7lEySBPwHGGtmB5nZEQS/Z22S2zNXFQ+4cSIpGzgOuIow4Ep6RdLZ5eoMlfR9SemSHpE0SdJ0SdeE2/tJ+kDSS8CMsOw/4WhmlqRB5dq6StKX4ejkGUlPhOXNJf0rbHuSpOMS9ynETS6wHoLPWdLocNQ7Q1L/skqS7pH0haT3gEOS1dk4ORnYbmZPlxWY2SIz+5OkTEnPh5/Hp5JOAqiiPEvSsPB37xUgKzmHtP/zS3vj53zgbTP7UtI6Sb2BYcCPgLck1QdOAa4jCMqFZnakpAbAx5LeCds5CuhuZgvD1z82s3WSsoBJkv4FNAB+BfQGNgHvA5+F9f8IDDazcZLaAaOArnE+9njIkjQNyARaEQQcgK3ABWa2UVI+8ImkkQSfxUDgcILf86nAlMR3O24OJTimilwPYGY9wumsdyR1rqL8OqDIzHpK6llFu66WPODGz4XAY+HzYeHrXwGPh0H1TIKvg8WSTgd6SvpBWD8P6ARsByaWC7YAN5Wbp2sb1msJfGhm6wAkvQp0DuucCnTTN9eb50rKMbNNdXu4cVdsZr0AJPUFXpTUneBy+gcknQjsBFoDBcAJwAgzKwrfMzI53U4MSU8CxxP8ziwF/gRgZnMkLSL4fTi+kvITgcfD8umSpif+CL4dPODGgaRmBCOw7pIMSAcMuB0YA5xBMNJ9uewtwI1mNmqPdvoBW/Z4fSrQ18yKJI0hGPFVlb0jLaxfXNvjShVmNj4czTYHzg5/HmFmOyR9RfCZQPCZ769mAd8ve2Fm14efyWRgWSXvqer3ZH/+rFKGz+HGxw+AF83sQDNrb2ZtgYUEI4xhwJUEI7CyADsKuE5SPQBJnSU1qqDdPGB9GGy7AMeE5ROB70hqIimDcv8RgXeAG8peSOpVZ0eZJOGxpwNrCT6TVWGwPQk4MKw2FrggnJ/MAb6bnN7GzftApqTrypU1DH+OBS6G4HcJaAd8EWN5d6BnAvr/reQj3Pi4EHhoj7J/ARcBNwEvAiPNbHu47a9Ae2BqePZ5NcEc8J7eBq4Nv/J9AXwCYGbLJD0ATAC+Bj4HCsP33AQ8Gb4ng+A/17V1cIyJVjaHC8FI7XIzK5X0D+B1SZOBacAcADObGp4AmkaQ9e2jZHQ6XszMJJ0PDJZ0O8HvzBbgDuA14GlJM4AS4Aoz2ybpz5WUPwU8H/6OTCP4A+7iwK80209IyjazzeEIdwTwnJmNSHa/nHPf8CmF/cd94QhwJsH0xX+S3B/n3B58hOuccwniI1znnEsQD7jOOZcgHnCdcy5BPOC6faJvsnfNlPSqpIbVv6vStvpJeiN8fp6kO6uo21jST/dhHxVmC6usfI86Q8tdBRjLvtpLmlnTPrr9nwdct6/Ksnd1J7icdLe1vQrU+PfLzEaa2Z5rmMtrDNQ44DqXCjzgurrwEdAxHNnNDhfYTwXaSjpdQZ7WqeFIOBtA0pmS5kgaB3yvrCFJV+ibTGcFkkYoyIH7mYIcuA8BB4ej60fCer/QN5nW7i/XVo2yhUm6OmznMwUZ1sqP2k+V9JGCjGznhvUrzPLmXGU84LpaCS+0OIswfSRBYHvRzA4nuPLpl8CpZtab4Dr/WyVlAs8QXG57AkHynYo8TpCU5zCC7F+zgDuB+eHo+hdh4p9OBFnVegFHSDpRUll+2MMJAvqRMRzOv83syHB/swmyuJVpD3wHOIfgaq1MymV5C9u/WlKHGPbjvqX80l63r8pfavsR8CxwALDIzD4Jy48BuhGkmwSoD4wHugALzWwugKS/A4PY28nAZQBmVgoUSmqyR53Tw8en4etsggCcQ82zhXWX9FuCaYtsvsl1ATDczHYCcyUtCI+hsixvX8awL/ct5AHX7atd6RLLhEF1S/ki4F0zu3CPer2ou+xUAh40s7/ssY+b92EfQ4HzzewzSVcA/cpt27Mto/Isb+1ruF/3LeFTCi6ePgGOk9QRQFLDMEvVHKCDpIPDehdW8v7RBMmxy+ZLcwkSrOeUqzMK+HG5ueHWklqwb9nCcoDlCrK2XbzHtgGS0sI+H0SQPCjWLG/OAT7CdXFkZqvDkeLLCpKuA/wyvAvGIOBNSWsI7vvWvYImfgYMkXQVUApcF+bC/ThcdvXfcB63KzA+HGFvBi7Zx2xhvyLIuLaIYE66fGD/AviQILn5tWa2VVKsWd6cAzyXgnPOJYxPKTjnXIJ4wHXOuQTxgOuccwniAdc55xLEA65zziWIB1znnEsQD7jOOZcgHnCdcy5B/j/rex3CZlZQ1AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "\n",
    "tuned_parameters = [{'var_smoothing': [1e-6, 1e-07, 1e-08, 1e-09, 1e-10, 1e-11, 1e-12, 1e-13, 1e-14, 1e-15]}]\n",
    "\n",
    "clf = GridSearchCV(GaussianNB(), tuned_parameters, scoring='f1_macro')\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "#Evaluation\n",
    "\n",
    "print(\"Best parameters set found on development set:\\n\", clf.best_params_)\n",
    "print(\"\\n Best Score\", clf.best_score_) \n",
    "\n",
    "print(\"\\n Grid scores on development set:\")\n",
    "means = clf.cv_results_['mean_test_score']\n",
    "stds = clf.cv_results_['std_test_score']\n",
    "for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "    print(\"%0.3f (+/-%0.03f) for %r\" % (mean, std, params))\n",
    "\n",
    "print(\"Detailed classification report:\")\n",
    "print(\"The scores are computed on the full evaluation set. \\n\")\n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(classification_report(y_true, y_pred))\n",
    "\n",
    "#Lets print confision matrix\n",
    "labels =['Average','Bad','Good']\n",
    "disp = plot_confusion_matrix(clf, X_test, y_test,\n",
    "                                 display_labels=labels,\n",
    "                                 cmap=plt.cm.Blues)\n",
    "\n",
    "print ('\\nTesting score: ',clf.score (X_test, y_test)) # print the testing score(accuracy)\n",
    "print ('\\nTraining Score: ',clf.score (X_train, y_train)) # print the trainig score(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['fit_time', 'score_time', 'test_f1_macro']\n",
      "test_f1_macro [0.53084722 0.5399705  0.55384074 0.55624569 0.55094091]\n"
     ]
    }
   ],
   "source": [
    "#Cross Validate\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.metrics import recall_score\n",
    "\n",
    "scoring = ['f1_macro']\n",
    "scores = cross_validate(clf, X, Y, cv=5, scoring=scoring)\n",
    "print(sorted(scores.keys()))\n",
    "print('test_f1_macro', scores['test_f1_macro'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.45579826472607415 0.7046019793738675 -0.28825772682504214\n",
      "  -0.10797909629417606]\n",
      " [-0.45821755567357747 -0.7059423617884181 -0.2916635351593642\n",
      "  -0.09416709513907746]\n",
      " [-0.21141491550725078 -0.0031579275616275076 0.8725130077194068\n",
      "  -0.3905247495057231]\n",
      " ...\n",
      " [-0.4581169085501422 0.717465384690377 -0.2917362036100829\n",
      "  -0.07432632961038373]\n",
      " [0.751538661872047 -0.0025887537747943763 -0.10000450010468652\n",
      "  -0.05167456137188623]\n",
      " [-0.4595648294413432 0.7112979984399956 -0.2918739685292939\n",
      "  -0.09370938251503302]] ['Bad' 'Bad' 'Bad' ... 'Average' 'Bad' 'Average']\n",
      "4800 4800 1200 1200\n"
     ]
    }
   ],
   "source": [
    "#Load Kbest data\n",
    "arr2= np.load('data_pca.npy', allow_pickle= True)\n",
    "X= arr2[:,0:-1]\n",
    "Y= arr2[:,-1]\n",
    "\n",
    "\n",
    "### Split arrays or matrices into random train and test subsets - 20% for Test, 80% to train\n",
    "X_train, X_test, y_train, y_test= train_test_split(X, Y, test_size=0.20,  random_state = 200)\n",
    "\n",
    "print(X_train,y_train)\n",
    "print(len(X_train),len(y_train),len(X_test),len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters set found on development set:\n",
      " {'var_smoothing': 1e-06}\n",
      "\n",
      " Best Score 0.4601396309088952\n",
      "\n",
      " Grid scores on development set:\n",
      "0.460 (+/-0.018) for {'var_smoothing': 1e-06}\n",
      "0.460 (+/-0.018) for {'var_smoothing': 1e-07}\n",
      "0.460 (+/-0.018) for {'var_smoothing': 1e-08}\n",
      "0.460 (+/-0.018) for {'var_smoothing': 1e-09}\n",
      "0.460 (+/-0.018) for {'var_smoothing': 1e-10}\n",
      "0.460 (+/-0.018) for {'var_smoothing': 1e-11}\n",
      "0.460 (+/-0.018) for {'var_smoothing': 1e-12}\n",
      "0.460 (+/-0.018) for {'var_smoothing': 1e-13}\n",
      "0.460 (+/-0.018) for {'var_smoothing': 1e-14}\n",
      "0.460 (+/-0.018) for {'var_smoothing': 1e-15}\n",
      "Detailed classification report:\n",
      "The scores are computed on the full evaluation set. \n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Average       0.46      0.28      0.35       405\n",
      "         Bad       0.53      0.49      0.51       392\n",
      "        Good       0.49      0.72      0.58       403\n",
      "\n",
      "    accuracy                           0.50      1200\n",
      "   macro avg       0.50      0.50      0.48      1200\n",
      "weighted avg       0.50      0.50      0.48      1200\n",
      "\n",
      "\n",
      "Testing score:  0.48276523868797655\n",
      "\n",
      "Training Score:  0.46904203445488185\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEKCAYAAABewe3GAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd8FVXawPHfk5tGKoEk9F5FpFtQVxEb9l5Y165YQNd1LejqK/a2q66KKHZcFF0BZV1XUQGxgdIRKVKlB0gICQmkPe8fMwkXSLkhuWXw+fq5n2TOnDtzZrw8OfeZM2dEVTHGGBN8UeFugDHG/F5YwDXGmBCxgGuMMSFiAdcYY0LEAq4xxoSIBVxjjAkRC7jGGBMiFnCNMSZELOAaY0yIRIe7AZGqYaPG2rRF63A3I2JJuBvgAVsLisLdhIiWt2UDu3bk1Omj5Etpo1pSGFBdLdzyuaoOqsv+6soCbhWatmjNGxOmhLsZEStKLOTW5I2568LdhIg2cfgldd6GlhQS1+XigOrumjcyvc47rCMLuMYYDxMQ72RGLeAaY7xLgChfuFsRMAu4xhhv81B6ywKuMcbDLKVgjDGhYz1cY4wJAcF6uMYYExpiPVxjjAkZG6VgjDGhYBfNjDEmNARLKRhjTMhYD9cYY0LBUgrGGBMaAvjsopkxxoSG5XCNMSYULKVgjDGhYz1cY4wJEevhGmNMCIjd2muMMaFjt/YaY0wo2EUzY4wJHUspGGNMCNh8uMYYEyqWUjDGmNCxi2bGGBMilsM1xpgQEEspGGNM6FgP1xhjQkMs4BpjTPA5T9ixgGuMMcEngkRZwP3de/qlicyYs5SGqYm8/o9b9lv/5TfzGffxNwA0iI/ltuvOokPbZnXaZ1FxCU++OJ5lKzeQkpzA/bddTNPMNGYtWM5rY7+gpKSE6Ohobrj8VHp3b1+nfdXVky9NYMZs5/y8+cyt+63/bf0Wnhw5gV9XbeDawSdzydnH1nmfRcUlPP7ChxXn54G/XOKcn/nLGT12MiUlpURH+7jx8lPpc1iHOu+vrv7YpwWHNk0mb3cJT3y1fL/18dFRXNGvJWkJMUSJMOXXrcz8bXud9pkQ4+OqI1rRKCGG7IJi3vzxNwqLy+jXMpUTO2cAUFRSxvvzNrBhx6467au+eKmHG9TLeyJynoioiHQN5n4i0akDevP4vVdUub5ZZhrPjriW1/4+jD9dMIBnRk8KeNubsnK4fcTr+5X/b8pskhIb8M4Lf+GCM/rz6tjJAKQmJ/DI3Zfx2j9u4e6h5/P4Cx/W/oDq2aABvXnyb1dWuT45qQG3XHMGF59V+0C7KSuH2x54bb/yT6fMJjmpAWNfvJ2LzjyaV/71OQCpKQk8NvxPvPHMLdwz7IKIOD8AM9fkMOq71VWu/0P7xmzK282TU1bwwjerOPewpvgCDD4d0xO5rE+L/cpP6pzOsi35PPLFryzbks/JbpDdVlDE89+s5Mkpy/lsaRaX9m5+QMcUDCIS0CsSBHs8xWDgW+DSum5IRLwzuhno0a0tKUkNqlx/aJfWJLvru3VqxZZtuRXrvpg+j5vveZkhd47kmdEfU1pWFtA+v5+1hFMG9ALg+KMOZc7PK1FVOrVrTnqjFADatsqkqLiEouKSAz20etGzW7tqz09aahJdO7YkOnr/j+gX0+dx0/BRXHfHi/zjlY8oLQ3s/Hz302JOPb43EPnnB2DFtgIKikurqaHEuecnNjqKgqJSylQBGNgpnb8OaM/dAztyWtfMgPd5WLMUflzj9JJ/XLOdw5o552VVdiGFxc55Xp1dQMMGMQdwRMFRXwFXRFqJyFQRWSwii0Tkz275CBFZLyLz3Nfpfu+5R0SWi8hSETm1pn0ELeCKSBJwDHAtbsAVkff3aexbInKBiPhE5GkR+UlEFojIDe76Ae4JeBdY6JZ9JCKz3RMyxG9b14rIMhGZJiKvisiLbnmGiIx3t/2TiBwTrGM+UP+bMpsjencGYM26LKZ9/zPPP3w9o58eii8qiq++mR/QdrZm7yCzcSoAPp+PxIQ4duQV7FVn+sxFdGrXjNgYb2aT1qzLYur3C3nhkSG89vdhREVF8eW3tTg/6XvOT1Jl52fGIjp65PxMX5lN0+Q4Hj6tC/ec2JHxCzaiQNfMJDISY/nHtJU8NWU5rdLi6dA4IaBtJsdFs2O388dmx+4SkuP2Pw/926SxeHNefR7KgZNavGpWAvxVVQ8BjgKGikg3d92zqtrLfX0K4K67FDgUGAS8VFPHMJifqnOBz1R1mYhki0gfYBxwCfCpiMQCJwI34QTlXFU9XETigO9EZLK7nSOA7qq6yl2+RlWzRaQB8JOIjAfigPuBPkAeMAUo/1f4T5yT9a2ItAY+Bw4J4nHXytyfV/K/qbN57qHrK5Z/XbWBm+95GYDdRcU0TEkE4P+efpdNWTkUl5SStTWXIXeOBOD80/sz6IQ+qNu72YvfX/bVazfz6tjJPPW3q4J7UEE0Z+FKlq3cwI3DRwFQVFRCWqpzfu5/aiwbs3IoKSll89ZcrrvjRQAuOKM/p53Ql8pOj//5WbV2M6PHfs5T910V7MOoF4dkJrEudxcvfLua9MRYhh7TlienLKdLZhJdM5O46wQnDx0XHUVGUhwrthVw+/HtiY4S4qKjSIj1VdSZtGgzS7Lya9xnp/REjmqbxnPTV9VYNxSE+ksXqOpGYKP7e56ILAb2z7vscQ4wTlV3A6tEZDlOvPqhqjcEM+AOBp5zfx/nLt8PPO8G1UHAdFUtFJFTgB4icqFbPxXoBBQBP/oFW4BbReQ89/dWbr2mwNeqmg0gIv8GOrt1TgK6+f1PSRGRZFXd70+022MeAtCkecs6HXwgVqzZxD9e+YjH77mC1GSnB6IKpxzfi+v+eMp+9R+684+Ak6N86qUJPDPi2r3WZzROJWtbLhmNUyktLWVnwe6Kr+1btuXyf39/j+FDL6B500ZBPrLgUZRTj+/N9Zftf34evusywDk/T4wcz3MPXrfX+ozGKWRt3XN+8vc9P0+/y/BhF9KiaePgH0g9OLJNGl8s2wLA1p1FbCsoIjM5DgG+WLaF71fn7PeeZ75eCTg53CNbN2TsnPV7rc/bXUKK28tNiYsmb/ee1ErzlDgG927BqB9WU1BUXaojtKKi6v+Luoi0BXoDM3G+qQ8TkSuAWTi94BycYDzD723rqD5AByelICKNgYHAayKyGrgTp2e7G5gGnOoujyt/C3CLX5e9naqW93B3+m13AE4A7a+qPYG5QDzVf2GIcuuXb7tFZcEWQFVHq2o/Ve3XsFH6gRx6wDZv3c6Iv7/HPcMupFXzPfvqfVh7ps9YRE6u09vYkV/A5i2BXXnu37crk6fNA+DrGYvofWg7RIT8nYXc+8Q7XDf4ZLp3bVP/BxNCfbp34Gv/85NXwKYt+weWyhzdryuffz0XcM9P9/YV52f44+9w3R9P4TAPnZ+cgiK6ZCQBkBznIzMpjm07i1iSlc9RbdKI9Tn/vFPjo0mKDewSyM+bdnBEm4YAHNGmIQs37gAgrUEM1x7Zmndmr2VLflEQjubA1SKHmy4is/xeQ6rYXhIwHrhNVXcAo4AOQC+cHvA/yqtW8vbKvkdVCFYP90JgjKreUF4gIl8Dx+IE2euAfsBV7urPgZtEZIqqFotIZ2DvP72OVCBHVQvckQ9HueU/As+KSBpOSuEC3JwvMBkYBjzttqOXqs6rtyOtwiPPfcD8X1aRm1fAJTc+zZUXD6S0xOkVnHXKEbzz4TR25Bfwz9f+A4DPF8WoJ26ibctMrr70JO5+5G3KVIn2+bj12jNpktGwxn2ePrAPj784nstveZbkpAbcd9vFAHz02Uw2bMrmX+On8a/x0wB48r4rSUtNCs7BB+Dh595n3iLn/Fx0w1NcdfHAiotfZ59yBNk5edwwfBQFhbsRET787/e89eyttG2VyTWXnsSdD7+FquLz+bjturNompFW4z7PGNiXx174kMuGPUNKUgPu/8slAEz8bAYbNm3jnQ+n8s6HUwF4+v6rwnp+AK7s15KOGYkkxUbz0KAufLo4C5/7T/y71Tl8tnQLf+rTkuEDO4LApEWb2FlUypKsfJokx3H78c7Qv92lZbwzax35AfRKv1i2lasPb8VRbdLIKSjmzR/XAjCoawaJsdFc1NMZnVCm8PdpK4Jz4LUReH4WYKuq9qt2cyIxOMF2rKpOAFDVzX7rXwU+cRfX4XzLLtcS2FDt9ivN+9WRiEwDnlDVz/zKbsXJnd4KbAImqerV7roo4BHgLJzTtwUnB9wbuENVz3TrxQEf4XTblwIZwAhVneb+tboD54AXA9mq+jcRSQdGuvuOxklj3FjTMXQ9rLe+MWFKXU/FQSsqQobZRLI35q4LdxMi2sThl7BlxaI6fZCi09trwzMfC6jutrcHz64u4IrTDX4bJ3bc5lfezM3vIiJ/AY5U1UtF5FDgXZy8bXPgK6CTqlb5ly0oPVxVHVBJ2fN+i433WVcG3Ou+/E1zX+X1dgOnVbHbd1V1tIhEAxNxerao6lac9IUx5iBTnxfNcHK1lwMLRaT8W/C9wGAR6YWTLlgN3ACgqotE5APgF5wRDkOrC7ZwcN1pNkJETsLJ6U7G6QkbYw5y9XVrr6p+S+UJik+rec+jwKOB7uOgCbiqeke422CMCTHx1q29B03ANcb8PlnANcaYELGAa4wxIVDPF82CzgKuMcbbvBNvLeAaYzxMgnNrb7BYwDXGeJqlFIwxJlS8E28t4BpjvM16uMYYEwKBPs0hUljANcZ4mgVcY4wJEXtMujHGhIj1cI0xJhRs8hpjjAkNYa/ngEY8C7jGGA+zUQrGGBMyUXbRzBhjQkAspWCMMSEhWA/XGGNCxnq4xhgTInbRzBhjQsFyuMYYExqC2ATkxhgTKtbDNcaYELEcrjHGhILlcI0xJjScuRS8E3Et4BpjPM1D8dYCrjHG2+xOM2OMCQWbD/fgEBcdRduMxHA3I2Ld/OGCcDch4g07pm24mxDRpjWIqfM2bD5cY4wJGZsP1xhjQsZD8dYCrjHGw8QumhljTEjYOFxjjAkhC7jGGBMiHoq3eGdeM2OMqYSIBPQKYDutRGSqiCwWkUUi8me3vJGIfCEiv7o/09xyEZHnRWS5iCwQkT417cMCrjHGu9zJawJ5BaAE+KuqHgIcBQwVkW7AcOArVe0EfOUuA5wGdHJfQ4BRNe3AAq4xxrOcCcgDe9VEVTeq6hz39zxgMdACOAd42632NnCu+/s5wBh1zAAaikiz6vZhOVxjjKdFBSGJKyJtgd7ATKCJqm4EJyiLSKZbrQWw1u9t69yyjVVt1wKuMcbTahFv00Vklt/yaFUdvf/2JAkYD9ymqjuqyf9WtkKra0CVAVdEUqp7o6ruqG69McYEm9Ru8pqtqtqv+u1JDE6wHauqE9zizSLSzO3dNgOy3PJ1QCu/t7cENlS3/ep6uItworX/0ZQvK9C6ug0bY0wo1NeNZuJE7teBxar6jN+qScCVwBPuz4/9yoeJyDjgSCC3PPVQlSoDrqq2qmqdMcZEinq8tfcY4HJgoYjMc8vuxQm0H4jItcBvwEXuuk+B04HlQAFwdU07CCiHKyKXAu1V9TERaYmTRJ5dmyMxxpj6JjgjFeqDqn5L5XlZgBMrqa/A0Nrso8ZhYSLyInACTuQHJ5K/XJudGGNMsERJYK9IEEgP92hV7SMicwFUNVtEYoPcLmOMqVmAd5FFikACbrGIROEOdxCRxkBZUFtljDEB8lC8DSjgjsQZJpEhIg8CFwMPBrVVxhgTACE4Nz4ES40BV1XHiMhs4CS36CJV/Tm4zTLGmMAcjBOQ+4BinLSCzb9gjIkItZiYJiIEMkrhb8B7QHOcOyneFZF7gt0wY4wJRJRIQK9IEEgP909AX1UtABCRR4HZwOPBbJgxxgQiMkJpYAIJuGv2qRcNrAxOc4wxpnYOimFhIvIsTs62AFgkIp+7y6cA34amecYYUzVnlEK4WxG46nq45SMRFgH/9SufEbzmGGNMLUhgk4tHiuomr3k9lA0xxpgDcVCkFMqJSAfgUaAbEF9erqqdg9guY4ypkddSCoGMqX0LeBPn2E4DPgDGBbFNxhgTsPp6am8oBBJwE1T1cwBVXaGq9+HMHmaMMWEnAb4iQSDDwna7M6GvEJEbgfVAZg3vMXWw8rcsbn1oTMXy2o3buO3qQcxZtIZVa52ne+zILyQlqQGfvHZHuJpZJzcc05Y+LVPZsauEOz9etN/6xFgfNxzTlibJcRSXKi9/t4p123fVaZ/RUcLQP7SjXeME8neX8M+vV7Ilv4jDmqUwuG8Lon1CSakydtY6Fm3Kq9O+6kOnzCTSEmMpLi1j7m/b91ufkRxHy7QGAJSWKSuy8tlZVFqnfYpA5ybJJMVFU1JWxpKNeewuKaNhQgxtGyciAqqwautOcguL67Sv+iACPg/lFAIJuH8BkoBbcXK5qcA1wWxUdUSkFFiI80erFBimqt/X4v0jgHxV/XtwWlh37VtnVgTS0tIyjr7oQU459jCuvvD4ijqPvfQxyYnxVW0i4n29fCufL85i6B/aVbr+3B7NWJNdwDNTV9A8NZ5rjmzNI5OXBbTtjKRYbjq2HQ99tnSv8hM6pZNfVMJtE36mf7s0/ti3Jf/8eiV5u4t5+qvl5BQW07JhPPee3Jmb/72gzsdYV5t37GJDbiGdmyRXun5XcSkL1uVSWqakJcTQsUkS89fmBrTtuOgoOjdJZuH6ves3SYmnpKyM2WtySE+KpW16Iks35VFcWsYvG3ZQVFpGQqyPQ1uk8NOqnDofY32IlHRBIAKZvGam+2seeyYhD6dCVe0FICKn4tzxdnz1b/Gu7+f8SuvmjWnRtFFFmary32nz+dczN4WxZXWzZHM+GUlVT6vcIjWejxduAmBD7i4ykmJJjY8md1cJx7ZvxKBDmhDtE5Zv2cnrM9ag1T4r1dGvdUM+nOc842/m6hyuPtJ5LN/q7MKKOuu27yLGF0V0lFBSFsBGg2jHrhLioqvO+uXtKtmrbqxf3YzkOJo3jEdEyNtVzIqsnQHts3FiLL9lFwCwNb+IDplJAOzcvafnXFBUSpRIRW833DwUb6u98WEi1TzyV1XPD0qLaicFyIGKRxt/DKQBMcB9qvqxu+5vwBU4z5DfgnNrsid8MmUuZ53Ye6+ynxasJD0tiXYtM8LUquD7LaeQI9o0ZGlWPh3SE0lPiqNRYiyJcdH0b9eIBz5dQqkq1xzVmmPbN+abFdtq3GajhFi27SwCoEyhsKiU5Lho8nbvCVxHtkljdXZB2INtbTVNiSdnp/MVv0GMj4ykOBaszUWBDhmJZCbHkZW3u8btxEZHsbtkz3TXJaW63x+fxkmx7NxdEhnBlsiZJyEQ1fVwXwxZK2qngfuAt3igGTDQLd8FnOc+Rz4dmCEik4A+wKVAb5zjnYNHAm5RcQlffb+IO68/Y6/y/0yZy1kn9glTq0Lj44UbufKI1jxxdjfW5hSyOruA0jLlsGbJtGucwKNnHQJArC+KHW5P7/YTOpCZHEd0lJCeGMsTZ3cD4H+/bObr5ZUHZPXrU7RsGM8f+7bgsS9+DfLR1a/UBjE0SXUCLEDDhBgS4330bN0QcIZNFZc6x3lIs2TiYnxEAXExPnq5dTZsLyRrR80BOSHWR9vGiSzaEFjqIug8NltYdTc+fBXKhtSCf0qhPzBGRLrj5HQfE5HjcJ5I0QJoAvwBmOg3+c6kqjYsIkOAIQAtWob/KfBfz1zCoZ1bkN5oTw6vpLSUz79ZwMev3B7GlgVfYXEZL3+3umL5hQsPY0v+bg5pksz05dsYN2f9fu95ZuoKoOocbnZBEY0TY8kuKCZKoEGsj3z3q3KjhBj+ekJHRn67ms0B9AQjRUKsj46ZSSzakLtXLzRrx27WbCvYr/7ijc7FwKpyuEUlZcRFR1Hk9nKjfXt6t7HRURzSLIVlm/PYVRw5D33xUg7X03PbquoPQDqQAVzm/uzrBuTN7LlRI6AvP6o6WlX7qWq/xunpwWhyrfxnyhzOGrh3T/a72cvo0CqTZhkNw9Sq0EiI9VVcfR7YKZ3Fm/IoLC7j5407OLJtGinxTl8hMdZHemJgj9ibvXY7x3VsDMCRbdNY5AafhFgfd5/UiffmrGNZVn4QjiY44qoIgNsLi0lPiiPG55y/6CipNhfsL3tnEZkpcQCkJ8WyvcBJU/iihEObp7B62869csfhJoBPJKBXJAh0AvKIJCJdcSZH34YzeiJLVYtF5ASgjVttOvCWiDyBc7xnAa+Eo721UbiriO9mL+PR2y/aq/yTKfMOinTCLce1o1vTZJLjoxl5UQ8+nLehIsB+uXQLLVLjufkP7ShTZf32Xbzi9nbX5+7igznrufeUzs4wlTLljZm/sdXNzVZn6q9bGfqHdjx3fnfyd5fy/NdOj/jUrpk0SY7j/J7NOb9ncwAem7ysIlURLl2aJpPaIIZon3B42zR+yy6o6M1tyt1Fq0YJxPik4sKWqjJ/bS6FRaWs2baTQ1ukIjgXtlZsyd8rN1uVTTt20aVJMn3bpFUMCwNonhpPfIyPVo0SaNUoAYBF63MrUhXh5KFRYYgGmPkWkThVDft3Lb9hYeD8gbtXVf/r5m3/g3PBbB5wDHCaqq72u2i2BlgH/FLTsLCevfvq/6b+EKzD8LybPwz/sKlIN+yYtuFuQkS7+aKTWPbzvDqFyyYdu+tlz3wYUN1nzzlktqr2q8v+6iqQuRSOAF7H6UG2FpGewHWqekuwG1cZVfVVUb4V6F/FukdxxhAbYw4iziN2vNPFDSSx8zxwJs7XdlR1PnZrrzEmQkRJYK9IEEgON0pV1+zzV6Ru9w8aY0w98VAHN6CAu9ZNK6iI+IBbgMDusTTGmCASINpDETeQgHsTTlqhNc5Qqy/dMmOMCTsPxduA5lLIwrlTyxhjIopE0CPQAxHIKIVXqeTGAVUdEpQWGWNMLXgo3gaUUvjS7/d44DycSWCMMSbsImUEQiACSSm8778sIu8AXwStRcYYEyDh4JuAfF/t2HPbrDHGhE8EjbENRCA53Bz25HCjgGxgeDAbZYwxgZKIeWJZzaoNuO6zzHriPMcMoEwDnXzBGGOC7KB6TLobXCeqaqn7smBrjIkoXrq1N5C5FH4UEe/PB2iMOSiJSECvSFBlwBWR8nTDsThBd6mIzBGRuSIyJzTNM8aYqjmPSQ/sVfO25A0RyRKRn/3KRojIehGZ575O91t3j4gsd2PjqYG0t7oc7o84zwM7N5ANGWNMONTjnWZv4TzLccw+5c/uO3+2iHTDuQP3UKA58KWIdFbVaif2qi7gCoCqrqhlo40xJiTq86KZqk4XkbYBVj8HGOc+lGGViCwHjgCqfWpBdQE3Q0SqfFKhqj4TYMOMMSZoatHBTReRWX7Lo1V1dADvGyYiVwCzgL+qag7OQ2pn+NVZ55ZVq7qA6wOSwEOD3IwxvzNCVOAhausBPGJnFPAwzr0IDwP/AK6h8rhY4yiu6gLuRlV9qJaNM8aYkBGCO3mNqm6u2Jczkdcn7uI6oJVf1ZbAhpq2V921O+vZGmMimziPgQ/kdUCbF2nmt3geUD6CYRJwqYjEiUg7oBPOQINqVdfDPfGAWmiMMSFSnz1cEXkPGICT610HPAAMEJFeOOmC1cANAKq6SEQ+AH4BSoChNY1QgGoCrqpm1/UAjDEm2OprWJiqDq6k+PVq6tf6aeAHMluYMcZEjAi5iSwgFnCNMZ4lBDY/QaSwgGuM8S6p1zvNgs4CrjHGs5w7zSzgGmNMSHgn3FrANcZ4nIc6uBZwjTFeFjlz3QbCAq4xxrNslIIxxoSQXTQ7CPiihLTEmHA3I2KNu6q2ky79/qQdPizcTYhou1fWONdLzQRLKRhjTChYSsEYY0LIerjGGBMi3gm3FnCNMR4mgM96uMYYExoeircWcI0xXiaIh5IKFnCNMZ5mPVxjjAkBZ1iYdyKuBVxjjHeJ9XCNMSZk7NZeY4wJAWcC8nC3InAWcI0xnmajFIwxJkQ8lFGwgGuM8Tbr4RpjTAhYDtcYY0JFxEYpGGNMqHgn3FrANcZ4mJNS8E7ItYBrjPE074RbC7jGGK/zUMS1gGuM8TRLKRhjTIh4J9xawDXGeJ2HIq4FXGOMZwl2p5kxxoSGzYdrjDGh46F4awHXGONlgnioixsV7gYYY0xdiAT2qnk78oaIZInIz35ljUTkCxH51f2Z5paLiDwvIstFZIGI9AmkrRZwjTGeJbV4BeAtYNA+ZcOBr1S1E/CVuwxwGtDJfQ0BRgWyAwu4xhhvq6eIq6rTgex9is8B3nZ/fxs41698jDpmAA1FpFlN+7CAa4zxNAnwvwPURFU3Arg/M93yFsBav3rr3LJq2UWzCNXznAdISojDFxVFtC+KKWPu4pp732D5miwAcvMLSU1qwPSxw2vY0sGpx9n/t+f8REcxdczdAIx+fxqvfjCdaF8UJx/bnYduPbeGLUWmFk0aMmrEFWQ2TqFMlbcnfscr46btVSc1uQEv3v8n2rVMZ1dRMbc8PJbFKzbWab+xMdGMevByenVtTXbuTq659w3WbsxmwBFdeWDY2cTGRFNUXML/Pf8R38xaVqd91ZdaXDNLF5FZfsujVXX0ge62kjKt6U0RG3BFpAnwLHAUkAMUAU+p6sQ6bncacIeqzqqpbrhNGnUrjRsmVSy/8dg1Fb/f99wEUpIahKNZEeM/L/95r/PzzaxlfPr1Qr597x7iYmPYkp0XxtbVTUlJGfc9N4EFS9eRlBDH1DF3M23mEpau2lRR569Xn8rCZeu4/K5X6dSmCU/ffTHn3vxCQNtv1awRLz1wOWfd+M+9yi8/pz+5Owrpe/6DnH9yX0bccg7X3vsm27bnM/j2V9i0NZdDOjTjw+eHcugZ99XrMR+Q2o3D3aqq/Wq5h80i0kxVN7opgyy3fB3Qyq9eS2BDTRuLyJSCOOM8PgKmq2p7Ve0LXIpzUL97qspHX87lglP6hrspEeWN8d9w25UnExcbA0BGo+Qwt+jAbd62gwVL1wGQX7CbZas30Syj4V51urRryvSflgLw65rNtG7WqOKYLz7tcL586w6mjx3Os/dcSlSAz6FevumcAAANsklEQVQ57bgevPffmQB8PGUuxx/eBYCFy9axaWsuAItXbCQ+NobYmMjorwU5pTAJuNL9/UrgY7/yK9zRCkcBueWph+pEZMAFBgJFqvpyeYGqrlHVF0QkXkTeFJGFIjJXRE4AqKa8gYiMc4duvA94olsowAW3jOSEK57irYnf7bXuh7kryGyUTIfWmZW/+XdARDh/2IsMuPxJ3prwLQDL12Txw7wVnHTV05wx5DnmLFoT5lbWj1bNGtGjS0tmL1q9V/nPv67nzBN6AdCnWxtaNW1E88yGdG7bhPNO7sOga5/huMueoLSsjIsGHR7QvppnprJ+cw4ApaVl7MgvpFFq4l51zh7YiwXL1lJUXFL3g6sjoV6Hhb0H/AB0EZF1InIt8ARwsoj8CpzsLgN8CqwElgOvAjcH0t7I+BO1v0OBOVWsGwqgqoeJSFdgsoh0rqb8JqBAVXuISI9qthtR/vfa7TTLSGVLdh7nD3uRzm2acHSfjgCMnzyb80/9ffduP3vtLzTLaMiW7DzOG/Yindo2paS0jO15BXzx5h3M+WUNV9/7BvM+GuGpgfH7SmwQy5gnr+OeZ8aTt3PXXuuee/sLHv/rhUwfO5xflm9gwbJ1lJaWcfzhXejZtTVTxtwFQHxcDFuy8wF456nradOiMTHRPlo2bVRxDeDlcdN49z8zKo1M/onJru2bMuKWczh/2MjgHPABqK//u6o6uIpVJ1ZSV3FjTm1EasDdi4iMBI7FyeOuA14AUNUlIrIG6Oyur6z8OOB5t3yBiCyoZj9DcMbU0ap166AdTyCaZaQCztfiMwb0ZPYvazi6T0dKSkr5ZNp8prx9Z1jbF27lX68zGiVz5oAezFm0mhaZDTnrhJ6ICH0PbUuUCNu255Oe5s3UQrQvirefvJ5/fzaLT6bO32993s5dDHvoXxXL8z9+kDUbttG/d0fG/XcmD42ctN97Lr/rVaDqHO6Gzdtp0SSNDVnb8fmiSElqQE7uTgCaZzbknaeGcNMD77B6/db6PNS68dDf00hNKSwCKu7cUNWhOH9lMqj69FZ32mu8eujuZ7Sq9lPVfunpGYG2td7tLNxd0ZvZWbibqTOXcEgHZ4jftJ+W0qlNE1o0SQtb+8Jt3/MzZcYSDunQnNMH9GD6T86V8+VrNlNUXLLXRTWveeH+y1i2ehMvvTul0vUpSQ2IifYBcMW5R/P93OXk7dzF9J+WcvbAXqSnOcfeMCWBVk0D+7x89s1CBp9xJADnDOxdcT5Tkhrw/rM38tDIScxcsLKuh1avotwn99b0igSR2sOdAjwmIjepavkdHAnuz+nAZcAUN2XQGlgaQPlUEekO9AjdYRyYLdl5XH6n0xMpKS3jwlP7cVL/bgBMnDz7d3+xbMu2PP7k9tRKS0q5YFA/Tjq6G0XFJQx7aCz9L3mU2Bgfo0Zc7tl0wlE923PpGUey6Nf1FV/7Hx45iZZNGwHw5oRv6dKuKaNGXE5pWRlLV23ilofHArB01SYeffkTJrw4jCgRiktKufOpD1i7KafG/b7z8fe8/OAVzJ7wADk7dnLt394E4PqLj6NdqwzuvG4Qd17n3Ix1/rAX2ZqTH4zDrxUv/R8WJxURedwhGM8CRwJbgJ3AyzhXCV8G+gIlwO2qOlVE4qsobwC8CXQD5gEdgVtrGhbWp28//W7GT0E5toOBVwNZKKUdPizcTYhou5d+QFlBVp0+SN179tEJk78NqG6XpomzD2BYWL2K1B5u+V0dl1ax+qpK6u+qorywmu0YYzzMJiA3xphQsQnIjTEmdDwUby3gGmO8zFsTkFvANcZ4mofirQVcY4x31WJy8YhgAdcY420eirgWcI0xnmbDwowxJkQsh2uMMaEgEOBUvxHBAq4xxuO8E3Et4BpjPKt8AnKvsIBrjPE0D8VbC7jGGG+zHq4xxoSI3dprjDEh4p1wawHXGONhgT6RN1JYwDXGeJrdaWaMMaHinXhrAdcY420eircWcI0xXhY5j0APhAVcY4xnee1Os6hwN8AYY34vrIdrjPE0L/VwLeAaYzzNhoUZY0wo2I0PxhgTGl67aGYB1xjjaZZSMMaYELEerjHGhIiH4q0FXGOMx3ko4lrANcZ4loCnbu0VVQ13GyKSiGwB1oS7HX7Sga3hbkSEs3NUvUg7P21UNaMuGxCRz3COKxBbVXVQXfZXVxZwPUJEZqlqv3C3I5LZOaqenZ/ws7kUjDEmRCzgGmNMiFjA9Y7R4W6AB9g5qp6dnzCzHK4xxoSI9XCNMSZELOAGkYicJyIqIl3D3RavE5FSEZknIvNFZI6IHF3L948QkTuC1b5wEJEmIvKuiKwUkdki8oOInFcP250mIjaaIQgs4AbXYOBb4NK6bkhEfHVvjqcVqmovVe0J3AM8Hu4GhZOICPARMF1V26tqX5zPWcvwtsxUxwJukIhIEnAMcC1uwBWR90XkdL86b4nIBSLiE5GnReQnEVkgIje46weIyFQReRdY6JZ95PZmFonIEL9tXSsiy9zeyasi8qJbniEi491t/yQix4TuLARNCpADznkWka/cXu9CETmnvJKI/E1ElorIl0CXcDU2SAYCRar6cnmBqq5R1RdEJF5E3nTPx1wROQGgmvIGIjLO/ey9DzQIzyEd/OzW3uA5F/hMVZeJSLaI9AHGAZcAn4pILHAicBNOUM5V1cNFJA74TkQmu9s5Auiuqqvc5WtUNVtEGgA/ich4IA64H+gD5AFTgPlu/X8Cz6rqtyLSGvgcOCTIxx4MDURkHhAPNMMJOAC7gPNUdYeIpAMzRGQSzrm4FOiN8zmfA8wOfbOD5lCcY6rMUABVPcxNZ00Wkc7VlN8EFKhqDxHpUc12TR1ZwA2ewcBz7u/j3OX7gefdoDoI5+tgoYicAvQQkQvd+qlAJ6AI+NEv2ALc6pena+XWawp8rarZACLyb6CzW+ckoJvsud88RUSSVTWvfg836ApVtReAiPQHxohId5zb6R8TkeOAMqAF0AT4AzBRVQvc90wKT7NDQ0RGAsfifGbWAS8AqOoSEVmD83k4tory44Dn3fIFIrIg9Efw+2ABNwhEpDFOD6y7iCjgAxS4C5gGnIrT032v/C3ALar6+T7bGQDs3Gf5JKC/qhaIyDScHl91s3dEufUL63pckUJVf3B7sxnA6e7PvqpaLCKrcc4JOOf8YLUIuKB8QVWHuudkFrC+ivdU9zk5mM9VxLAcbnBcCIxR1Taq2lZVWwGrcHoY44CrcXpg5QH2c+AmEYkBEJHOIpJYyXZTgRw32HYFjnLLfwSOF5E0EYnG7x8iMBkYVr4gIr3q7SjDxD12H7AN55xkucH2BKCNW206cJ6bn0wGzgpPa4NmChAvIjf5lSW4P6cDl4HzWQJaA0sDLO8O9AhB+3+XrIcbHIOBJ/YpGw/8EbgVGANMUtUid91rQFtgjnv1eQtODnhfnwE3ul/5lgIzAFR1vYg8BswENgC/ALnue24FRrrvicb5x3VjPRxjqJXncMHpqV2pqqUiMhb4j4jMAuYBSwBUdY57AWgezqxv34Sj0cGiqioi5wLPishdOJ+ZncDdwMfAyyKyECgBrlLV3SLyUhXlo4A33c/IPJw/4CYI7E6zg4SIJKlqvtvDnQi8oaoTw90uY8wellI4eIxwe4A/46QvPgpze4wx+7AerjHGhIj1cI0xJkQs4BpjTIhYwDXGmBCxgGsOiOyZvetnEfm3iCTU/K4qtzVARD5xfz9bRIZXU7ehiNx8APuodLawqsr3qfOW312AgeyrrYj8XNs2moOfBVxzoMpn7+qOczvpXmN7xVHrz5eqTlLVfccw+2sI1DrgGhMJLOCa+vAN0NHt2S12B9jPAVqJyCnizNM6x+0JJwGIyCARWSIi3wLnl29IRK6SPTOdNRGRieLMgTtfnDlwnwA6uL3rp916d8qemdYe9NtWrWYLE5Hr3e3MF2eGNf9e+0ki8o04M7Kd6davdJY3Y6piAdfUiXujxWm400fiBLYxqtob586n+4CTVLUPzn3+t4tIPPAqzu22f8CZfKcyz+NMytMTZ/avRcBwYIXbu77TnfinE86sar2AviJynIiUzw/bGyegHx7A4UxQ1cPd/S3GmcWtXFvgeOAMnLu14vGb5c3d/vUi0i6A/ZjfKbu11xwo/1ttvwFeB5oDa1R1hlt+FNANZ7pJgFjgB6ArsEpVfwUQkX8BQ9jfQOAKAFUtBXJFJG2fOqe4r7nuchJOAE6m9rOFdReRR3DSFknsmesC4ANVLQN+FZGV7jFUNcvbsgD2ZX6HLOCaA1UxXWI5N6ju9C8CvlDVwfvU60X9zU4lwOOq+so++7jtAPbxFnCuqs4XkauAAX7r9t2WUvUsb21ruV/zO2EpBRNMM4BjRKQjgIgkuLNULQHaiUgHt97gKt7/Fc7k2OX50hScCdaT/ep8DlzjlxtuISKZHNhsYcnARnFmbbtsn3UXiUiU2+b2OJMHBTrLmzGA9XBNEKnqFren+J44k64D3Oc+BWMI8F8R2Yrz3LfulWziz8BoEbkWKAVucufC/c4ddvU/N497CPCD28POB/50gLOF3Y8z49oanJy0f2BfCnyNM7n5jaq6S0QCneXNGMDmUjDGmJCxlIIxxoSIBVxjjAkRC7jGGBMiFnCNMSZELOAaY0yIWMA1xpgQsYBrjDEhYgHXGGNC5P8BZsoL9q/m05sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "\n",
    "tuned_parameters = [{'var_smoothing': [1e-6, 1e-07, 1e-08, 1e-09, 1e-10, 1e-11, 1e-12, 1e-13, 1e-14, 1e-15]}]\n",
    "\n",
    "clf = GridSearchCV(GaussianNB(), tuned_parameters, scoring='f1_macro')\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "#Evaluation\n",
    "\n",
    "print(\"Best parameters set found on development set:\\n\", clf.best_params_)\n",
    "print(\"\\n Best Score\", clf.best_score_) \n",
    "\n",
    "print(\"\\n Grid scores on development set:\")\n",
    "means = clf.cv_results_['mean_test_score']\n",
    "stds = clf.cv_results_['std_test_score']\n",
    "for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "    print(\"%0.3f (+/-%0.03f) for %r\" % (mean, std, params))\n",
    "\n",
    "print(\"Detailed classification report:\")\n",
    "print(\"The scores are computed on the full evaluation set. \\n\")\n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(classification_report(y_true, y_pred))\n",
    "\n",
    "#Lets print confision matrix\n",
    "labels =['Average','Bad','Good']\n",
    "disp = plot_confusion_matrix(clf, X_test, y_test,\n",
    "                                 display_labels=labels,\n",
    "                                 cmap=plt.cm.Blues)\n",
    "\n",
    "print ('\\nTesting score: ',clf.score (X_test, y_test)) # print the testing score(accuracy)\n",
    "print ('\\nTraining Score: ',clf.score (X_train, y_train)) # print the trainig score(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['fit_time', 'score_time', 'test_f1_macro']\n",
      "test_f1_macro [0.47942543 0.46982902 0.47266061 0.46363159 0.47202167]\n"
     ]
    }
   ],
   "source": [
    "#Cross Validate\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.metrics import recall_score\n",
    "\n",
    "scoring = ['f1_macro']\n",
    "scores = cross_validate(clf, X, Y, cv=5, scoring=scoring)\n",
    "print(sorted(scores.keys()))\n",
    "print('test_f1_macro', scores['test_f1_macro'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}