Chapter 4 Naive Bayes Exercise.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 521,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "from getpass import getpass\n",
    "import json\n",
    "import time\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from textblob import TextBlob\n",
    "from sklearn.feature_extraction.text  import CountVectorizer\n",
    "from collections import Counter, OrderedDict\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pull NYT articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = 'https://api.nytimes.com/svc/search/v2/articlesearch.json?'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "········\n"
     ]
    }
   ],
   "source": [
    "api_key = getpass()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [],
   "source": [
    "sections = ['Arts','Business','Obituaries','Sports','World']\n",
    "# number of articles per section\n",
    "# do 500 so it doesn't take too long\n",
    "n_articles = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Arts\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "Business\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "Obituaries\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "Sports\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "World\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n"
     ]
    }
   ],
   "source": [
    "final_df = pd.DataFrame()\n",
    "for section in sections:\n",
    "    print(section)\n",
    "    for page in range(1, int(n_articles/10)+1):\n",
    "        print(page)\n",
    "        query = f'api-key={api_key}&fq=news_desk:(\"{section}\")&page={page}'\n",
    "        full_url = base_url + query\n",
    "        content = requests.get(full_url).json()\n",
    "\n",
    "        articles_df = pd.DataFrame(content['response']['docs'])\n",
    "        articles_df['headline'] = [d.get('main') for d in articles_df['headline']]\n",
    "        # this version of the api doesn't have the body, so combine abstract and lead paragraph\n",
    "        articles_df['snippet'] = articles_df['abstract'] + ' ' + articles_df['lead_paragraph']\n",
    "        cols = ['web_url','headline','snippet','section_name']\n",
    "        articles_df = articles_df[cols]\n",
    "        final_df = final_df.append(articles_df)\n",
    "    \n",
    "        time.sleep(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>web_url</th>\n",
       "      <th>headline</th>\n",
       "      <th>snippet</th>\n",
       "      <th>section_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.nytimes.com/interactive/2020/11/27...</td>\n",
       "      <td>The Best of Late Night This Week</td>\n",
       "      <td>The late-night hosts took the holiday off, and...</td>\n",
       "      <td>Arts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.nytimes.com/interactive/2020/11/25...</td>\n",
       "      <td>The Myth of North America, in One Painting</td>\n",
       "      <td>How Benjamin West remade a bloody battle as a ...</td>\n",
       "      <td>Arts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.nytimes.com/interactive/2020/11/25...</td>\n",
       "      <td>What’s in Our Queue? ‘Down by Law’</td>\n",
       "      <td>I’m the deputy culture editor for news. I’m do...</td>\n",
       "      <td>Arts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.nytimes.com/interactive/2020/11/20...</td>\n",
       "      <td>The Best of Late Night This Week</td>\n",
       "      <td>This week, the late-night hosts crunched the n...</td>\n",
       "      <td>Arts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.nytimes.com/interactive/2020/11/19...</td>\n",
       "      <td>What’s in Our Queue? ‘The Great’</td>\n",
       "      <td>I’m the editor of The Times’s Arts &amp; Leisure s...</td>\n",
       "      <td>Arts</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             web_url  \\\n",
       "0  https://www.nytimes.com/interactive/2020/11/27...   \n",
       "1  https://www.nytimes.com/interactive/2020/11/25...   \n",
       "2  https://www.nytimes.com/interactive/2020/11/25...   \n",
       "3  https://www.nytimes.com/interactive/2020/11/20...   \n",
       "4  https://www.nytimes.com/interactive/2020/11/19...   \n",
       "\n",
       "                                     headline  \\\n",
       "0            The Best of Late Night This Week   \n",
       "1  The Myth of North America, in One Painting   \n",
       "2          What’s in Our Queue? ‘Down by Law’   \n",
       "3            The Best of Late Night This Week   \n",
       "4            What’s in Our Queue? ‘The Great’   \n",
       "\n",
       "                                             snippet section_name  \n",
       "0  The late-night hosts took the holiday off, and...         Arts  \n",
       "1  How Benjamin West remade a bloody battle as a ...         Arts  \n",
       "2  I’m the deputy culture editor for news. I’m do...         Arts  \n",
       "3  This week, the late-night hosts crunched the n...         Arts  \n",
       "4  I’m the editor of The Times’s Arts & Leisure s...         Arts  "
      ]
     },
     "execution_count": 306,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.to_csv('raw_articles_output.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "World            501\n",
       "Obituaries       500\n",
       "Arts             500\n",
       "Sports           500\n",
       "Business Day     335\n",
       "Technology       146\n",
       "Your Money        14\n",
       "Times Insider      1\n",
       "Reader Center      1\n",
       "Travel             1\n",
       "New York           1\n",
       "Name: section_name, dtype: int64"
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df['section_name'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combining to business\n",
    "final_df.loc[~final_df['section_name'].isin(['World','Obituaries','Arts','Sports']),'section_name'] = 'Business'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "World         501\n",
       "Obituaries    500\n",
       "Arts          500\n",
       "Sports        500\n",
       "Business      499\n",
       "Name: section_name, dtype: int64"
      ]
     },
     "execution_count": 309,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df['section_name'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean text\n",
    "punctuations = string.punctuation\n",
    "punctuations = punctuations + \"’‘“”—\"\n",
    "stopw = stopwords.words('english')\n",
    "stopw.extend(['new','york','time','2'])\n",
    "\n",
    "final_df['combined_text'] = final_df['headline'] + ' ' + final_df['snippet']\n",
    "final_df['combined_text'] = final_df['combined_text'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [],
   "source": [
    "def removeStopwords(texto):\n",
    "    \n",
    "    blob = TextBlob(texto).words\n",
    "    outputlist = [word for word in blob if word not in stopw\n",
    "                                        and word not in punctuations]\n",
    "    text_cleaned = ' '.join(word for word in outputlist)\n",
    "    text_cleaned = text_cleaned.replace('-',' ')\n",
    "    return text_cleaned\n",
    "\n",
    "\n",
    "def tokenize(text):\n",
    "    stem = nltk.stem.SnowballStemmer('english')\n",
    "    text = text.lower()\n",
    "\n",
    "    for token in nltk.word_tokenize(text):\n",
    "        if token in string.punctuation: continue\n",
    "        yield stem.stem(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df['combined_text_cleaned'] = final_df['combined_text'].apply(removeStopwords)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implement Bernoulli Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_df = final_df[['combined_text_cleaned','section_name']].copy()\n",
    "model_df = model_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "world = model_df[model_df['section_name']=='World'].copy().reset_index(drop=True)\n",
    "other = model_df[model_df['section_name']!='World'].copy().reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 855,
   "metadata": {},
   "outputs": [],
   "source": [
    "# includes more samples for world section since it has less vocab\n",
    "# than other sections and keep not getting predicted\n",
    "world_test_size = 0.50\n",
    "other_test_size = 0.20\n",
    "\n",
    "world_test = world.sample(n=int(world_test_size * len(world)))\n",
    "world_train = world[~world.index.isin(world_test.index)]\n",
    "\n",
    "other_test = other.sample(n=int(other_test_size * len(other)))\n",
    "other_train = other[~other.index.isin(other_test.index)]\n",
    "\n",
    "train = pd.concat([other_train, world_train])\n",
    "test = pd.concat([other_test, world_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 856,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sports        406\n",
       "Arts          401\n",
       "Business      399\n",
       "Obituaries    394\n",
       "World         251\n",
       "Name: section_name, dtype: int64"
      ]
     },
     "execution_count": 856,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['section_name'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 857,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "World         250\n",
       "Obituaries    106\n",
       "Business      100\n",
       "Arts           99\n",
       "Sports         94\n",
       "Name: section_name, dtype: int64"
      ]
     },
     "execution_count": 857,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['section_name'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 933,
   "metadata": {},
   "outputs": [],
   "source": [
    "class0 = 'World'\n",
    "alpha = 10\n",
    "beta = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 934,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using nltk\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "def vectorize(doc):\n",
    "    features = defaultdict(int)\n",
    "    for token in tokenize(doc):\n",
    "        features[token] += 1\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 935,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate priors\n",
    "theta_jc_df = pd.DataFrame()\n",
    "theta_c_df = pd.DataFrame()\n",
    "ndoc_by_sec = []\n",
    "for s in sections:\n",
    "\n",
    "    vectors = map(vectorize, train[train['section_name']==s]['combined_text_cleaned'])\n",
    "\n",
    "    # change to 0 or 1\n",
    "    class_df = pd.DataFrame(vectors).fillna(0)\n",
    "    class_df = class_df.where(class_df==0, 1)\n",
    "    ndoc_by_sec.append(len(class_df))\n",
    "    # calculate theta jc\n",
    "    # number of doc containing word j in section c over total number of docs in section c with laplace\n",
    "    #thetajc = ((class_df.sum() + alpha - 1)/(len(class_df) + alpha + beta -2))\n",
    "    thetajc = class_df.sum()\n",
    "    thetajc = pd.DataFrame(thetajc).T\n",
    "    thetajc['section'] = s\n",
    "    theta_jc_df = theta_jc_df.append(thetajc)\n",
    "    \n",
    "    thetac = len(class_df)/len(train)\n",
    "    theta_c_df = theta_c_df.append(pd.DataFrame([{'section':s,'thetac':thetac}]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 936,
   "metadata": {},
   "outputs": [],
   "source": [
    "ndoc_by_sec = np.array(ndoc_by_sec)\n",
    "ndoc_by_sec = np.expand_dims(ndoc_by_sec, axis=1)\n",
    "theta_jc_df = theta_jc_df.fillna(0)\n",
    "features = [c for c in theta_jc_df.columns if 'section' not in c]\n",
    "theta_jc_df.loc[:, features] = (theta_jc_df[features] + alpha - 1) / (ndoc_by_sec + alpha + beta -2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 937,
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of doc  in section c over total number of docs\n",
    "theta_jc_df = theta_jc_df.set_index('section')\n",
    "theta_c_df = theta_c_df.set_index('section')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 938,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>thetac</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>section</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Arts</th>\n",
       "      <td>0.216640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Business</th>\n",
       "      <td>0.215559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Obituaries</th>\n",
       "      <td>0.212858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sports</th>\n",
       "      <td>0.219341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>World</th>\n",
       "      <td>0.135602</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              thetac\n",
       "section             \n",
       "Arts        0.216640\n",
       "Business    0.215559\n",
       "Obituaries  0.212858\n",
       "Sports      0.219341\n",
       "World       0.135602"
      ]
     },
     "execution_count": 938,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta_c_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 939,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>best</th>\n",
       "      <th>late</th>\n",
       "      <th>night</th>\n",
       "      <th>week</th>\n",
       "      <th>host</th>\n",
       "      <th>took</th>\n",
       "      <th>holiday</th>\n",
       "      <th>seem</th>\n",
       "      <th>thank</th>\n",
       "      <th>earlier</th>\n",
       "      <th>...</th>\n",
       "      <th>discont</th>\n",
       "      <th>noteworthi</th>\n",
       "      <th>tel</th>\n",
       "      <th>aviv</th>\n",
       "      <th>gilkey</th>\n",
       "      <th>npr</th>\n",
       "      <th>zabihullah</th>\n",
       "      <th>tamanna</th>\n",
       "      <th>videograph</th>\n",
       "      <th>glimps</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>section</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Arts</th>\n",
       "      <td>0.072495</td>\n",
       "      <td>0.031983</td>\n",
       "      <td>0.051173</td>\n",
       "      <td>0.123667</td>\n",
       "      <td>0.036247</td>\n",
       "      <td>0.027719</td>\n",
       "      <td>0.027719</td>\n",
       "      <td>0.027719</td>\n",
       "      <td>0.023454</td>\n",
       "      <td>0.023454</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019190</td>\n",
       "      <td>0.019190</td>\n",
       "      <td>0.019190</td>\n",
       "      <td>0.019190</td>\n",
       "      <td>0.019190</td>\n",
       "      <td>0.019190</td>\n",
       "      <td>0.019190</td>\n",
       "      <td>0.019190</td>\n",
       "      <td>0.019190</td>\n",
       "      <td>0.019190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Business</th>\n",
       "      <td>0.032120</td>\n",
       "      <td>0.040685</td>\n",
       "      <td>0.038544</td>\n",
       "      <td>0.111349</td>\n",
       "      <td>0.023555</td>\n",
       "      <td>0.040685</td>\n",
       "      <td>0.032120</td>\n",
       "      <td>0.040685</td>\n",
       "      <td>0.021413</td>\n",
       "      <td>0.027837</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019272</td>\n",
       "      <td>0.019272</td>\n",
       "      <td>0.019272</td>\n",
       "      <td>0.019272</td>\n",
       "      <td>0.019272</td>\n",
       "      <td>0.019272</td>\n",
       "      <td>0.019272</td>\n",
       "      <td>0.019272</td>\n",
       "      <td>0.019272</td>\n",
       "      <td>0.019272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Obituaries</th>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.021645</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sports</th>\n",
       "      <td>0.063291</td>\n",
       "      <td>0.052743</td>\n",
       "      <td>0.099156</td>\n",
       "      <td>0.158228</td>\n",
       "      <td>0.029536</td>\n",
       "      <td>0.050633</td>\n",
       "      <td>0.023207</td>\n",
       "      <td>0.048523</td>\n",
       "      <td>0.023207</td>\n",
       "      <td>0.023207</td>\n",
       "      <td>...</td>\n",
       "      <td>0.018987</td>\n",
       "      <td>0.018987</td>\n",
       "      <td>0.018987</td>\n",
       "      <td>0.018987</td>\n",
       "      <td>0.018987</td>\n",
       "      <td>0.018987</td>\n",
       "      <td>0.018987</td>\n",
       "      <td>0.018987</td>\n",
       "      <td>0.018987</td>\n",
       "      <td>0.018987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>World</th>\n",
       "      <td>0.028213</td>\n",
       "      <td>0.037618</td>\n",
       "      <td>0.043887</td>\n",
       "      <td>0.072100</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.047022</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.028213</td>\n",
       "      <td>0.028213</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031348</td>\n",
       "      <td>0.031348</td>\n",
       "      <td>0.031348</td>\n",
       "      <td>0.031348</td>\n",
       "      <td>0.031348</td>\n",
       "      <td>0.031348</td>\n",
       "      <td>0.031348</td>\n",
       "      <td>0.031348</td>\n",
       "      <td>0.031348</td>\n",
       "      <td>0.031348</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 7964 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                best      late     night      week      host      took  \\\n",
       "section                                                                  \n",
       "Arts        0.072495  0.031983  0.051173  0.123667  0.036247  0.027719   \n",
       "Business    0.032120  0.040685  0.038544  0.111349  0.023555  0.040685   \n",
       "Obituaries  0.019481  0.019481  0.019481  0.021645  0.019481  0.019481   \n",
       "Sports      0.063291  0.052743  0.099156  0.158228  0.029536  0.050633   \n",
       "World       0.028213  0.037618  0.043887  0.072100  0.034483  0.047022   \n",
       "\n",
       "             holiday      seem     thank   earlier  ...   discont  noteworthi  \\\n",
       "section                                             ...                         \n",
       "Arts        0.027719  0.027719  0.023454  0.023454  ...  0.019190    0.019190   \n",
       "Business    0.032120  0.040685  0.021413  0.027837  ...  0.019272    0.019272   \n",
       "Obituaries  0.019481  0.019481  0.019481  0.019481  ...  0.019481    0.019481   \n",
       "Sports      0.023207  0.048523  0.023207  0.023207  ...  0.018987    0.018987   \n",
       "World       0.034483  0.034483  0.028213  0.028213  ...  0.031348    0.031348   \n",
       "\n",
       "                 tel      aviv    gilkey       npr  zabihullah   tamanna  \\\n",
       "section                                                                    \n",
       "Arts        0.019190  0.019190  0.019190  0.019190    0.019190  0.019190   \n",
       "Business    0.019272  0.019272  0.019272  0.019272    0.019272  0.019272   \n",
       "Obituaries  0.019481  0.019481  0.019481  0.019481    0.019481  0.019481   \n",
       "Sports      0.018987  0.018987  0.018987  0.018987    0.018987  0.018987   \n",
       "World       0.031348  0.031348  0.031348  0.031348    0.031348  0.031348   \n",
       "\n",
       "            videograph    glimps  \n",
       "section                           \n",
       "Arts          0.019190  0.019190  \n",
       "Business      0.019272  0.019272  \n",
       "Obituaries    0.019481  0.019481  \n",
       "Sports        0.018987  0.018987  \n",
       "World         0.031348  0.031348  \n",
       "\n",
       "[5 rows x 7964 columns]"
      ]
     },
     "execution_count": 939,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta_jc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 940,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate weights\n",
    "\n",
    "numer = ((np.array(theta_jc_df) * np.array(1 - theta_jc_df[theta_jc_df.index==class0]))) \n",
    "denom = ( np.array(theta_jc_df[theta_jc_df.index==class0]) * (1 - np.array(theta_jc_df)))\n",
    "\n",
    "wjc = np.log(numer/denom)\n",
    "\n",
    "w0 = (np.log((1 - np.array(theta_jc_df))/(np.array(1 - theta_jc_df[theta_jc_df.index==class0])))).sum(1) + \\\n",
    "np.log(np.array(theta_c_df)/np.array(theta_c_df[theta_c_df.index==class0]['thetac'])).T[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 941,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_vectors = pd.DataFrame()\n",
    "for s in sections:\n",
    "    vectors = map(vectorize, test[test['section_name']==s]['combined_text_cleaned'])\n",
    "    df = pd.DataFrame(vectors).fillna(0)\n",
    "    df = df.where(class_df==0, 1)\n",
    "    df['section_name'] = s\n",
    "    test_vectors = test_vectors.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 942,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_vectors = test_vectors.set_index('section_name')\n",
    "test_vectors = test_vectors.fillna(0)\n",
    "test_vectors = test_vectors.where(test_vectors==0, 1)\n",
    "\n",
    "missing_words = list(set(theta_jc_df.columns) - set(test_vectors.columns))\n",
    "for w in missing_words:\n",
    "    test_vectors[w] = 0\n",
    "test_vectors = test_vectors[list(theta_jc_df.columns)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 943,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_df = pd.DataFrame((np.matmul(wjc, np.array(test_vectors.T)).T) + w0, columns = sections)\n",
    "prediction_df['pred'] = prediction_df.idxmax(axis=\"columns\")\n",
    "prediction_df['actual'] = list(test_vectors.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 944,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "World         313\n",
       "Obituaries    106\n",
       "Arts           99\n",
       "Sports         94\n",
       "Business       37\n",
       "Name: pred, dtype: int64"
      ]
     },
     "execution_count": 944,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction_df['pred'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 945,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "World         250\n",
       "Obituaries    106\n",
       "Business      100\n",
       "Arts           99\n",
       "Sports         94\n",
       "Name: actual, dtype: int64"
      ]
     },
     "execution_count": 945,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction_df['actual'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 946,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy rate: 0.9029275808936826\n"
     ]
    }
   ],
   "source": [
    "print('accuracy rate:', accuracy_score(prediction_df['actual'], prediction_df['pred']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 947,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(91.68, 0.5, 'predicted label')"
      ]
     },
     "execution_count": 947,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATAAAAEwCAYAAAAw+y3zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3wUdf7H8dc7GxSkJSIiAhoULKiACqjYsIF6Yi/wO8/exXr2gqjn6VlPxVOxIXoqtvPAhlhABQvYwMKJCkiVlkBAFEg+vz9mggtsNpuY7Ozg5/l45JHdmd2Z967xw3e+M/P9ysxwzrk4yos6gHPO1ZQXMOdcbHkBc87Flhcw51xseQFzzsWWFzDnXGzlRx0g7sq/HROb61DO2frPUUdwrtoesKmqbJ23wJxzseUFzDkXW17AnHOx5QXMORdbXsCcc7HlBcw5F1tewJxzseUFzDkXW17AnHOx5QXMORdbXsCcc7HlBcw5F1tewJxzseUFzDkXW17AnHOx5QXMORdbXsBywJBhI+l97rUccs41PP7fNwCYNOVH+lxyE4f2u5azb7ibJT8vizjl2jr02psBk97ihsmj6HX52VHHqVKc8sYpK0SXd50uYJKOkGSStqlkfYGkc7KdK9m302bw3Ih3efaOa3jp3usZNe4Lps76iWvvGczFJx7NsIE3sv9uO/HIi69FGXMtysuj7303MPCgk7i+wwF07XsoLbdtF3WsSsUpb5yyQrR51+kCBvQF3gf6rLlCUgIoACItYD9Mn02nrbegQf31yU8k6Lr91rz5wadMmTmHrttvBUD3ztsxcuwnUcZcS1G3zsz9bhrzp0ynbMUKxj0znI6H9Yw6VqXilDdOWSHavOtsAZPUCNgdOJWwgEnqIekdSU8BE4FbgC0lfS7pNkktJb0bPv9S0p51nbP95q0Y/9W3FC9ewrJffuXd8ROZM38h7TdvxdsffQ7AiDHjmD1/YV1HqZbCVi0onj5r1fOSGbMpbNUiwkTpxSlvnLJCtHnX5Uk9DgdeN7NvJS2UtFO4vBuwvZlNkVQUPu4MIOmvwAgzuylsoW2QasOSzgDOALj/hks547jDahxyyzabctpRB3HqtbezQYP6bNO2DYm8PG46/xRuGvQU/3pmGPvu0pl6+Tn2n0prz7NglsPzm8Qpb5yyQqR5c+z/ilrVF/hn+PiZ8PkrwMdmNqWS94wDHpVUD3jJzD5P9SIzGwQMgtqZlejonntxdM+9ALhryAu0aFbIFm1a8siNfwVgysw5jB434ffuplYVz5hDYZtNVz0vaN2SkllzI0yUXpzyxikrRJt3nTyElNQM2Bd4WNJU4FLgOEDA0sreZ2bvAnsBM4EnJJ1Q92lhQcliAGbNXcDIsZ/wp713WbWsvLycB4YO57iDemQjSsamjfuCjdsX0ayoNYl69ejapzcTho2MOlal4pQ3Tlkh2rzragvsaGCImZ1ZsUDSaGCPNV5XCjROes3mwEwze0hSQ2AnYEhdh73g5vsoKV1CfiLBtWcfT9NGDRkybCRPvfI2AAfsthNH7r9m9GiVl5UxtF9/zh8xhLxEgrGPPsvsrydHHatSccobp6wQbV7l9LF1DUkaBdxiZq8nLTsfOBv43swOSVr+FNAReA34kqC1tgJYApyQ5nAT8Iltnatr6Sa2XScLWDZ5AXOubvnM3M65dZIXMOdcbHkBc87Flhcw51xseQFzzsWWFzDnXGx5AXPOxZYXMOdcbHkBc87Flhcw51xseQFzzsWWFzDnXGx5AXPOxZYXMOdcbHkBc87Flhcw51xsratDSmdNnAYJ/POuraKOkLF/fzgz6gjrrP43HhR1hFrjLTDnXGx5AXPOxZYXMOdcbHkBc87Flhcw51xseQFzzsWWFzDnXGx5AXPOxZYXMOdcbHkBc87Flhcw51xseQFzzsWWFzDnXGx5AXPOxZYXMOdcbHkBc87Flg9omGM69NqbY+/uT14iwZiHhzLiH/dHHWkVrbcene6/D9WrhxL5zH/nHX58+BG2ufEGGmy2GQD5jRuxsnQJn514UrRhU8jl73ZNcciq9RtQcMjx5DffFDBKhj9B/XbbU3+rjmBG2c+llAwbQvmSRXWWIZICJqkMmAgIKAP6mdnYGmznLOBnMxtSyxEjobw8+t53A3cfcDzFM+Zw5bhhTBg2ktnffBd1NABs+XIm9Duf8mXLUCJBxwfvp/iDD5l0bf9Vr2l7Xj/Kli6NMGVquf7dJotL1qa9juXX77+m+IWHIC+B6q3HknmzKR09HICGXfeh8Z4Hs+i1p+ssQ1SHkMvMrLOZdQKuBG6uyUbM7IF1pXgBFHXrzNzvpjF/ynTKVqxg3DPD6XhYz6hjraZ82TIAlJ9PXn4+mK22vvl++zL3jZFRREsrDt9thThk1Xr1WW+zdvz8+ZhgQXkZ9usybPkvv72m3np1niMX+sCaAMUAknpIerlihaSBkk4KH98i6WtJEyTdHi4bIOmS8PEoSf+Q9LGkbyXtGS5PSLpN0rjwvWeGy1tKelfS55K+lLRn+NrB4fOJki7K5hdR2KoFxdNnrXpeMmM2ha1aZDNC1fLy2PHxwez66ssUfzyO0q+/XrWqSedOLF9YzC8zZkQYMLVYfLehOGRNFG5E+dIlFPQ+geanXUXTPx2/qmA17nEoLc6/iQbbd1vVGqsrUfWBNZD0OVAfaAnsm+7FkjYEjgC2MTOTVFDJS/PNrJukg4HrgP2BU4FFZtZV0vrAGElvAEcCI8zsJkkJYAOgM9DKzLYP91vZfuqGtNYiW6OFE7nycj478SQSjRrR4Zab2WCLtvz8wxQANj7gAOaNzL3WFxCP77ZCDLIqL496LduwaMRQVsyaSpOex9Coey9KRw+ndNQwSkcNo1H3XjTs0oPSd1+ueoM1FPUh5DbAgcAQKcV/td8sBn4BHpZ0JPBzJa97Mfz9CVAUPu4JnBAWzI+AZkB7YBxwsqQBwA5mVgr8AGwh6V5JB4b7XYukMySNlzT+a0oz+8QZKJ4xh8I2m656XtC6JSWz5tba9mtT2ZIlLPr0Uwp33TVYkEjQrMfezHvzrWiDVSJO320cspYtLqFscQkrZk0F4JdvPqPeJm1We82yr8ZRf5sd6zRH5IeQZvYBsBHQHFjJ6pnqh69ZCXQDXgAOB16vZHO/hr/L+K11KeC8sGB2NrO2ZvaGmb0L7AXMBJ6QdIKZFQOdgFHAucDDlWQeZGZdzKxLBxrX5GOnNG3cF2zcvohmRa1J1KtH1z69mTAsd1o09QoKSDRqBEDe+utR0LUry6ZNA6CwaxeWTZvG8nnzooxYqVz/bpPFIWv50sWULS4msWFwaLt+261ZOX8OicLmq15Tv31HVi6YU6c5Ir+MQtI2QAJYAEwDOoSHevWB/YD3JTUCNjCzVyV9CFTndMwI4GxJb5vZCklbERStjYCZZvaQpIbATpJeBZab2QuSvgcG19bnzER5WRlD+/Xn/BFDyEskGPvos8z+enI2I6RVr1kztu5/DcrLA+Ux/+23WTgmOHncfP/9mTvyzYgTVi7Xv9tkccm6aMRQCg8/GSUSrCyZT8nwJyj40/HkN2sBVk7ZooWUvPZUnWZQFMfWSZdRQNBCusrMXgnX3QocBkwGlgPDCIrQfwmKmoDbzezx8PBviZndLmkUcImZjZe0ETDezIok5QF/A3qH751H0Io7HLgUWAEsAU4gOKHwGL+1Aq80s9fSfZazVJRbnRNp+MS2DuI3se2m19xfafdSJC0wM0ukWXcZcFmKVd1SvHZA0uMeSY/nE/aBmVk5cFX4k+zx8GdNO1Ua3DmXUyLvA3POuZryAuaciy0vYM652Kq0D0zSRCBVB7UAM7OOdZbKOecykK4T/5CspXDOuRqotICZ2bSKx5I2B9qb2ZuSGqR7n3POZUuVfWCSTgeeBx4MF7UGXqrLUM45l4lMOvHPBXYnvC/QzCYDG9dlKOecy0QmBexXM1te8URSPqk7951zLqsyKWCjJV1FMATOAcBzQN0O8uOccxnIpIBdQXD/4ETgTOBV4Jq6DOWcc5mo8myimZVLepxgLC0D/me5Nrqac+4PqcoCJulPwAPA9wQXsbaVdGZVozQ451xdy+R6rjuAfczsOwBJWwKvAF7AnHORyqQPbG5F8Qr9AOTW+LbOuT+kdPdCHhk+/CocqfRZgj6wYwjGk3fOuUhVOiKrpMfSvM/M7JS6iRQvcRqRNU7u+zFeV+qcu1nvqCOssx6wqdUfkdXMTq6bOM45VzsyOQtZn2Buxe0IZwkC8BaYcy5qmXTiPwFsAvQCRhPczF17kyE651wNZVLA2pnZtcBSM3sc+BOwQ93Gcs65qmVSwFaEv0skbQ805bdZr51zLjKZXMg6SFIhcC3BHI2NgP51mso55zKQyb2QD4cPRwNb1G0c55zLXLoLWS9O90Yzu7P24zjnXObStcAaZy2Fc87VQLoLWa/PZhDnnKsun9jWORdbXsCcc7HlBcw5F1t+FtI5F1uZnIXcGuhKcBErQG/g3boM5ZxzmajyLKSkN4CdzKw0fD6AYGo155yLVCZ9YJsBy5OeL8fvhXTO5YBMh9P5WNIASdcRTK82pG5j/XF16LU3Aya9xQ2TR9Hr8rOjjpNWNrJefdt97HH0KRx62kUp1//w40z6nncVnQ7qw6PP/rdW9rl8+QouvvFOep3Qj+P6XcHMOcEUEN98N4W+511F71Mv5PDTL+a1d8bUyv5SidPfAUSXt8oCZmY3AScDxUAJcLKZ/T2TjUtqLem/kiZL+l7S3ZLWk3SSpIGVvGds+LtI0v9l/lEyynODpP1rc5u1SXl59L3vBgYedBLXdziArn0PpeW27aKOlVK2sh7Rax8G3Vz5PMpNGzfiqnNP4eRjDq32tmfOmcuJF689LsELr71Fk8YNGTFkICcedQh3PPQkAA3qr8/Nl5/H8Ef+yaCbr+Hm+x9j8ZKl1d5vVeL0dwDR5s30MooNgMVmdjcwQ1Lbqt4gScCLwEtm1h7YimAki5vSvc/MuocPi4BaK2CSEmbW38zerK1t1raibp2Z+9005k+ZTtmKFYx7ZjgdD+sZdayUspW1S8cONG3cqNL1zQqbssM27chPJNZaN+zNdznu3Cs44sxLuO6uBykrK8ton2+PHcfhPXsA0HOv3fjws4mYGUWtN6WodUsANt5oQ5oVNGVhyeLqf6gqxOnvAKLNW2UBCw8bLweuDBfVA57MYNv7Ar+Y2WMAZlYGXAScQlAQ20h6XdL/wn1U7G9J+PAWYE9Jn0u6aM1Wm6SXJfUIH98vabykryRdn/SaqZL6S3ofOEbSYElHh+t2ljRa0ieSRkhqGS4/X9LXkiZIeiaDz1lrClu1oHj6rFXPS2bMprBVi2xGyFiuZ/1+2gxeHzWGJ+/+G/958HYSeXm8/NZ7Gb33pwUL2aT5RgDkJxI0brgBJYtXH4R4wqTJrFi5ks02rf3PnOvf7ZqizJvJeGBHADsCnwKY2SxJmdzovR3wSfICM1ss6cdwv92A7YGfgXGSXjGz8UkvvwK4xMwOAZB0Upp9XW1mCyUlgLckdTSzCeG6X8xsj3AbB4a/6wH3AoeZ2TxJxxG0DE8J99vWzH6VVJBqZ5LOAM4A2JMN6VBb971r7clXKps1KnI5nvXDzyby1eQfOPbcKwD49dflbFjQBIDzrruVGXPmsmLFSmbPnc8RZ14CwF+OOJgjD9w35ecQv33eeQuKueKWe7n5sn7k5dXBteA5/t2uJcK8mRSw5WZmkgxAUsMMty2CeSQrWz7SzBaE23wR2AMYn+L1mTg2LCr5QEugA1BRwIameP3WBMVzZHCkSwKYHa6bAPxb0kvAS6l2ZmaDgEFQu9OqFc+YQ2GbTVc9L2jdkpJZuTmHcK5nNTMOO6AHF5/257XW3Xv9ZUDQB3bVrQN5/M4bVlu/yUbNmDNvPps0b8bKsjJKl/5M0ybBYeySpT9z1tV/5/yT+9Cpw1Z1kj3Xv9s1RZk3k38+npX0IFAg6XTgTeDhKt4D8BXQJXmBpCZAG6CMtYtbVYVg5Rp564fbbAtcAuxnZh2BV0iaPQlI1csq4Csz6xz+7GBmFQftfwLuA3YGPpGUSZGvFdPGfcHG7YtoVtSaRL16dO3TmwnDRmZr99WS61l33WkH3njvAxYULwKgZHEpM3+al9F79+nehZfeGAXAG+9+wC6dt0cSy1es4LwBt3LYAXtz4N7d02/kd8j173ZNUebNZETW2yUdACwmaLn0N7NM0r0F3CLpBDMbEh7e3QEMJjhsPEDShsAy4HCCw7dkpaw+JtlU4BxJeUArgkNQgCYERWqRpBbAQcCoKrL9D2guaTcz+yA8pNwK+AZoY2bvhP1m/0dw4qEkg8/7u5WXlTG0X3/OHzGEvESCsY8+y+yvJ2dj19WWrayX3HQXH3/xFSWLStmnzxn0O/E4VqxcCUCf3r2Yt7CYY8+5nCU/LyNP4okXX2H4I/+k3eZtuOCkvpx2xY1YeTn5+flce95ptGrRvMp9HnXQflx+yz30OqEfBY0bcfvVwSUcr4/+gE8mfEPJ4iX8Jyxwf7/0XLZtV+U5rWqJ098BRJu30pm5V71A+oeZXV7Vskre2wb4F7ANQevpVYLWUl/gYKAh0A54KunK/yVm1igsKq8DGxEUvX8SnDzoDHwJtAAGmNkoSYOBXYAfgF+BYWY2WNJUoIuZzQ+3PRh42cyel9QZuIdgkpL8cPuDgXfCZQKeNLNb0n1Gn5m7bvjM3K5Cupm5Mylgn5rZTmssmxAerv3heQGrG17AXIV0BSzdaBRnA+cAW0qakLSqMTC29uI551zNpOsDewp4DbiZ4NKCCqVmtrBOUznnXAYqPQtpZovMbCpwN7DQzKaZ2TRghaRdshXQOecqk8llFPcDS5KeLw2XOedcpDIpYLKknn4zKyezC2Cdc65OZVLAfgjvD6wX/lxAcLmCc85FKpMCdhbQHZgJzCC43uqMugzlnHOZyORK/LlAnyxkcc65akl3HdhlZnarpHtJcZ+imZ1fp8mcc64K6Vpg34S/azpChHPO1al0sxIND38/nr04zjmXuXSHkMNJM8SNmVV/EHLnnKtF6Q4hbw9/Hwlswm/DSPclGNrGOecile4QcjSApBvNbK+kVcMl+czczrnIZXIdWHNJW1Q8CUdArXpUOOecq2OZ3BJ0ETBKUsXV90XAmXWWyDnnMlTlgIYAktYnGFUVYJKZ/VqnqWLEBzR0AAM/fSDqCBnrt9NZUUeolnQDGmYyL+QGwKVAPzP7AthM0iG1mM8552okkz6wx4DlwG7h8xnA3+oskXPOZSiTAralmd0KrAAws2VApU0655zLlkwK2HJJDQgvapW0JcHMP845F6lMzkJeRzC9WRtJ/wZ2B06qy1DOOZeJtAUsnES2kOBq/F0JDh0vqJhn0TnnopS2gJlZuaR+ZvYs8EqWMjnnXEYy6QMbKekSSW0kbVjxU+fJnHOuCpn0gZ0S/j43aZkBW6R4rXPOZU0mQ0q3zUYQ55yrrioLmKT6wDnAHgQtr/eAB8zslzrO5pxzaWVyCDkEKAXuDZ/3BZ4AjqmrUM45l4lMCtjWZtYp6fk7kr6oq0DOOZepTM5CfiZp14onknYBxtRdJOecy0wmLbBdgBMk/Rg+3wz4RtJEwMysY52lc865NDIpYAfWeQrnnKuBKg8hzWxaup9shPwj6dBrbwZMeosbJo+i1+VnRx0nrThlhdzP+8Srozjskps59JKbGfLqqNXWPTb8bbbrcwHFi5dEE64KUX23mfSB5QxJV0v6StIESZ+H/XG/d5s9JHWvjXy/l/Ly6HvfDQw86CSu73AAXfseSstt20UdK6U4ZYXczzt5+iyef/sDnrnpr7z4j8sY/elXTJs9F4DZ84sZO/F/tNyoMOKUqUX53camgEnaDTgE2Cnsd9sfmP47t5kP9AByooAVdevM3O+mMX/KdMpWrGDcM8PpeFjPqGOlFKeskPt5f5j5E53aF9Fg/fXITyTosm073hw3EYB/DPkPf/3zoShHh+GL8ruNTQEDWgLzK8bjN7P5ZjZL0lRJ/5D0cfjTDkDS5pLeCltrb0naLFw+WNKdkt4BhgJnAReFLbo9JR0j6UtJX2R7+rjCVi0onj5r1fOSGbMpbNUimxEyFqeskPt527VpyfhvvqekdCnLfl3Oe59/zZwFxbw9fiItNmzKNpu3ijpipaL8bjPpxM8VbwD9JX0LvAkMrZi7ElhsZt0knQD8k6ClNhAYYmaPSzoFuAc4PHz9VsD+ZlYmaQCwxMxuBwjPrvYys5mSCrL26YKdr7Uok0lXIhGnrJDzebdstQmnHrofp930Lzaovz5bb74pibw8Bv1nJA9dnXv9dauJ8LuNTQvMzJYAOwNnAPOAoZJOClc/nfS7Yuz+3YCnwsdPENwKVeE5MyurZFdjgMGSTgcSqV4g6QxJ4yWN/5rSmnyclIpnzKGwzaarnhe0bknJrLm1tv3aFKesEI+8R+27G8/fcilDBpxP04Yb0Kr5hsyct4AjL7uVA/pdz08LSzj6ytuYV7I46qirifK7jU0BAzCzMjMbZWbXAf2AoypWJb+ssrcnPV6aZh9nAdcAbYDPJTVL8ZpBZtbFzLp0oHG1PkM608Z9wcbti2hW1JpEvXp07dObCcNG1tr2a1OcskI88i5YFPxjOGv+Qt4cN4FD9+rGe4NuYuTA6xg58DpabFjA8zdfSvOCJhEnXV2U321sDiElbQ2Um9nkcFFnYBqwA3AccEv4+4Nw/VigD0Hr68/A+5VsuhRY9RchaUsz+wj4SFJvgkK2oHY/TWrlZWUM7def80cMIS+RYOyjzzL768lVvzECccoK8ch74Z2PUrJkKfmJBNecfDRNG20QdaSMRPndZjSxbS6QtDPBDeUFwErgO4LDyfEEU78dTNCi7Gtm30kqAh4FNiI45DzZzH6UNBh42cyeD7e7FfA8UA6cRzATeXuC4bPfAi60NF+ST2zrwCe2rUvpJraNTQvMzD4hxeUOCjoQ7zOz69d4/VRg3xTbOWmN598CybdDvff70zrnsiFWfWDOOZcsNi2wyphZUdQZnHPR8BaYcy62vIA552LLC5hzLra8gDnnYssLmHMutryAOediywuYcy62vIA552LLC5hzLra8gDnnYssLmHMutryAOediywuYcy62vIA552IrNiOy5iofkdXFzaOd1xrnM6ct/+zRSkdk9RaYcy62vIA552LLC5hzLra8gDnnYssLmHMutryAOediywuYcy62vIA552LLC5hzLra8gDnnYssLmHMutryAOediywuYcy62vIA552LLC5hzLra8gDnnYssLmHMutvKjDuBW16HX3hx7d3/yEgnGPDyUEf+4P+pIlYpTVohX3mxkbd2ikEdvPI1NmjWl3IyHXxjNwKffXO01e+28NS/cdR5TZ80H4KW3P+GmQcN/137Xq5fPYzeexo7bbs7CRUv58+X3M232Ajpt1YZ7r/4LTRo2oKysnFseeZnn3hiXdluxbIFJukvShUnPR0h6OOn5HZIuznBbgyUdnWJ5D0kv107izCgvj7733cDAg07i+g4H0LXvobTctl02I2QsTlkhXnmzlXVlWTmX3TmUjkddwx4n3MTZx+3Ltltsutbr3v9sMl37DKBrnwHVKl6bt2zGyIcuW2v5yYfvSXHpUjocdiX3/PsN/n7BMQD8/MtyTrn2YToffS2H9LuL2y/pS9NGDdLuI5YFDBgLdAeQlAdsBGyXtL47MKaqjUhK1Em6Girq1pm5301j/pTplK1YwbhnhtPxsJ5Rx0opTlkhXnmzlXXO/EV8PulHAJb8/AuTpsxm0+YFGb///w7elTFPXMO4ZwZw39UnkJdX6dD1q+ndY0eeGD4WgBfeHM8+3bYFYPKPP/Hdj3MBmD2vhHnFpTTfsHHabcW1gI0hLGAEhetLoFRSoaT1gW2BAkmfSZoo6dFwOZKmSuov6X3gmOSNSjpQ0qRw3ZHZ+ziBwlYtKJ4+a9XzkhmzKWzVItsxMhKnrBCvvFFk3bxlMzptvRkff/nDWut27bgl44dez7CBF9EhbKFt07Ylx/Tsxt4n30zXPgMoKy/n/w7eLaN9tdq4gBlzFgJQVlbOoiXLaFbQaLXXdNmuLevlJ/h++ry024plH5iZzZK0UtJmBIXsA6AVsBuwCPgWeBjYz8y+lTQEOBv4Z7iJX8xsDwiKVvi7PvAQsC/wHTC0sv1LOgM4A2BPNqQD6f+VyJjW/hcsZ2eNilNWiFfeLGdt2GB9ht5+Lpfc/jSlS39Zbd1nk6bR7uBLWbrsVw7cYweeu+s8tjvsSvbpti07dijigyevBaDB+usxb2EpAM/d0Y+iVhuxXr182myyIeOeGQDAvU+9yZBh76MqPt8mGzVl8N9O55T+D1f5uWNZwEIVrbDuwJ0EBaw7QQGbSVCkvg1f+zhwLr8VsFTFaRtgiplNBpD0JGGRWpOZDQIGQe1Oq1Y8Yw6FbX7rgyho3ZKSWXNra/O1Kk5ZIV55s5k1Pz/B0NvP5enXPuSltz9da31yQXv9/Yncc2WCZgWNkMSTw8dwzb0vrPWeY/46EAhadQ/fcCoHnH7rautn/FRM6002ZObcYhKJPJo2asDCRUsBaNywPv+950Kuu+9FPp64dmtwTXE9hITf+sF2IDiE/JCgBdYdWPu/xOqWVrI80n+Sp437go3bF9GsqDWJevXo2qc3E4aNjDJSpeKUFeKVN5tZB113MpOmzObuJ99Iub5FsyarHnfZri15EgtKlvDOx99wxP5daF4YHH0UNmnIZi2bZbTPl0d/zl96Bz1AR+3fhVHjJgFQLz/Bc3f048mXx/LCm+Mz2lbcW2B/BX4wszJgoaQCgj6x84DTJbUzs++AvwCjq9jeJKCtpC3N7Hugbx1mT6m8rIyh/fpz/ogh5CUSjH30WWZ/PTnbMTISp6wQr7zZytq9c3uOP6Q7E7+dvuow79qBL9Bmk6AQPfT8KI7cvwtnHrMPK8vKWfbLco6/8gEAvvlhFgPue5FX7/8reRIrVpZx/i1P8uPsBVXu97GX3mXw307n6//eTPHipRx/xYMAHNOzK3vutBXNChpxwqG7A+wrszwAAA1+SURBVHBa/0fSbiu2M3OHZxCLgXvM7Jpw2WBgNzPbWtJ+wO0ERXoccLaZ/SppKtDFzOYnvedlM3s+7A/7JzAfeB/Y3swOSZfDZ+Z2cbMuzcwd2xZY2Opqssayk5IevwXsmOJ9RWne8zpBX5hzLgbi3AfmnPuD8wLmnIstL2DOudjyAuaciy0vYM652PIC5pyLLS9gzrnY8gLmnIstL2DOudjyAuaciy0vYM652PIC5pyLLS9gzrnY8gLmnIstL2DOudjyAuaci63Yjsi6rpN0Rjh5SM6LU1aIV944ZYXs5/UWWO5KOSNSjopTVohX3jhlhSzn9QLmnIstL2DOudjyApa7YtPvQbyyQrzyxikrZDmvd+I752LLW2DOudjyAuaciy0vYM652PIC5v5QJOVJalL1K3ODpEJJHaPOkau8gOUISbtLahg+Pl7SnZI2jzpXZSRdIKmJAo9I+lRSz6hzpSLpqTBrQ+Br4H+SLo06V2UkjQrzbgh8ATwm6c6ocyWTNFzSsMp+spXDC1juuB/4WVIn4DJgGjAk2khpnWJmi4GeQHPgZOCWaCNVqkOY9XDgVWAz4C/RRkqraZj3SOAxM9sZ2D/iTGu6HbgDmAIsAx4Kf5YAX2YrRH62duSqtNLMTNJhwN1m9oikE6MOlYbC3wcT/E/2hSSle0OE6kmqR1DABprZCkm5fP1QvqSWwLHA1VGHScXMRgNIutHM9kpaNVzSu9nK4S2w3FEq6UrgeOAVSQmgXsSZ0vlE0hsEBWyEpMZAecSZKvMgMBVoCLwbHpovjjRRetcDI4DvzGycpC2AyRFnqkzzMB8AktoStMizwi9kzRGSNgH+DxhnZu9J2gzoYWY5eRgpKQ/oDPxgZiVhf01rM5sQcbSMSMo3s5VR50hF0u5mNqaqZblA0oEEV9//EC4qAs40sxHZ2L8fQuaOi8zs8oonZvajpO2iDFSF3YDPzWyppOOBnYC7I86UkqQWwN+BTc3sIEkdCPI/Em2ySt1L8H1WtSxyZva6pPbANuGiSWb2a7b27y2wHCHpUzPbaY1lE8wsJ0+hS5oAdAI6Ak8QFIMjzWzvSIOlIOk14DHgajPrJCkf+MzMdog42mok7QZ0By4E7kpa1QQ4wsw6RRIsBUlHpltvZi9mI4e3wCIm6WzgHGDLsChUaAzk3CFDkjiddNjIzJ4N+xgxs5WSyqIOlcJ6QCOC/y8bJy1fDBwdSaLK9U6zzgAvYH8QTwGvATcDVyQtLzWzhdFEykjFSYe/AHvm+EmHpZKaEfyPhaRdgUXRRlqbmY2W9D6wg5ldH3WedMzs5LAf9GgzezaqHF7AImZmiySVEvzRTos6TzUcR3DS4RQzmxOedLgt4kyVuRgYRtDKHUNwlizXWjQAmFlZeEIk55lZuaR+QGQFzPvAcoSkfwNXmtmPUWfJVHg5Qnsze1PSBkDCzEqjzpVK2O+1NcH1a/8zsxURR6qUpDuA9sBzwNKK5dnqV6oOSdcSXMg6lNWzZuXowQtYjpD0NtAV+Jjf/hDMzA6LLlXlJJ1OMP75hma2ZXgm6gEz2y/iaKtI2tfM3q6swzkXCwKApMdSLDYzOyXrYaogaUqKxWZmW6RYXuv8EDJ3JPd5CNgD6BtRlkycC3QDPgIws8mSNo420lr2Bt4mdYdz1jqaq8vMTo46Q6bMrG2U+/cCliPCDtzOBP1KxxLcY/ZAtKnS+tXMllfcPRQeouVUc97Mrgs7ml+LsqO5uiS1Jrjua3eC7/R94AIzmxFpsBTCW7TOBipuJxoFPJitQ3S/lShikraS1F/SN8BAYDrBof0+ZnZvxPHSGS3pKqCBpAMI+muGR5xpLWZWDvSLOkc1PUZw0mFToBXB95rqsDIX3A/sDPwr/Nk5XJYV3gcWMUnlwHvAqWb2Xbjsh2z1IdRU2LI5lWA0ChHcu/ew5eAfVNQdzdUl6XMz61zVslwg6Ys1L7BNtayu+CFk9I4C+gDvSHodeIbfRnrIWWHLpmIIlVxX0fl9btIyA3L1H4n54e1ZT4fP+wILIsyTTpmkLc3se4Dwxu6sXSTsLbAcEQ62dzjBH+u+wOPAf8zsjUiDVULS7sAAYHOCfwhFFs8+rcvCa+oGEtyvCcEdGRfk0nWCki4kyFVI8I9YxdnIIoJrA9/OSg4vYLknvJDxGOA4M9s36jypSJoEXAR8QtK/uGaWky0FSdsDHYD6FctydaSPOJB0O8F9m9sC3wIzCf4WHjOzWVnL4QXM1YSkj8xsl6hzZELSdUAPggL2KnAQ8L6Z5eTV+OFh2N3ArgSHuh8QjFbyQ9o3RkDSekAXgmK2W/hTYmYdsrF/PwvpauodSbdJ2k3SThU/UYeqxNHAfsCc8BqrTsD60UZK6ymC23NaEpyJfI7f+sNyTQOC0TKahj+zCK8NzAbvxHc1VdH66pK0zAj673LNsvC+vZUKZiSaS+524ENwZPRE0vMnw3sOc4akQcB2QClBwRoL3GlmxdnM4QXM1YiZ7RN1hmoYL6mAoLP5E4KJJz6ONlJa70i6guCMtBHcOP9KxU3eOXL5x2YErdjJBP1fM4CSbIfwPjBXLZKON7MnJV2car2Z5dT0X2uSVAQ0yeWhr5PuL6z4nzP5spqcOdMbTuKyHUH/V3dge2Ah8IGZXZeNDN4Cc9XVMPzdOO2rcoikvVItM7OszZ6TCUldgekV9xeGA0QeRTAhyYAcaXmtEl60/KWkEoLx1RYBhxDcI5uVAuYtMLfOk5R8i1N9gv/BPsm1S1QkfQrsb2YLw6L7DHAeweQp2+bSWVNJ5xO0unYHVhBcE/ZB+HtieKFznfMWmKsRSbcCfyO4Red1gjN7F5rZk5EGS8HMVhuNQlIb4NaI4qSTSGplHQcMMrMXgBckfR5hrlSKgOcJLu+YHVUIv4zC1VRPC2aPPoSgA3cr4NJoI2VsBkF/Ta5JhKN6QHDZR/LV7DnV2DCzi83s+SiLF+TYl+JipWL8+4OBp8PDnijzVErSvfzWIV4xn+UX0SWq1NMEo3zMJ2jZvgcgqR05OIZ/LvAC5mpqeHg70TLgHEnNgV8izlSZ8UmPVxIU3Jyb8cnMbpL0FsEFrG8kjeyRR9AX5tbgnfiuxiQVAovDiSg2ILg8YU7UudYk6QIzu7uqZS5+vIC5GpF0QqrluXiDdCWTBn9mZjtGlcnVDj+EdDXVNelxfYJO50+BnClgkvoSDNHdVtKwpFWNyd3xtVw1eAFzNWJmq/XJSGoKPFHJy6MyFpgNbATckbS8FMjZK/Fd5vwQ0tWKcHKHCWa2bdRZ3B+Ht8BcjYRXtyefJetAhDM0pyLpfTPbI5z5PPlf6orRY5tEFM3VEm+BuRqRtHfS05XAtFyc9sut27yAud9N0kbAglyckahCONjiHoTzLJrZZxFHcrXAbyVy1SJpV0mjJL0oaUdJXwJfAj9JOjDqfKlI6k8wSUozgg79wZKuiTaVqw3eAnPVImk8cBXB8MGDgIPM7ENJ2xBc4Z5z11aFkwbvaGa/hM8bAJ/6CYf48xaYq658M3vDzJ4jGGP+QwAzmxRxrnSmkjQbEcFIot9HE8XVJj8L6aoreZynZWusy6nmfNJN3L8CX0kaGa7aH3g/smCu1vghpKsWSWXAUoJLERoAP1esAuqbWb3K3ptt4YimEOSsR1B8ywgLr5k9HlE0V0u8gLl1Vnhx7U3AKcA0gi6TNsBjwFVmtiLCeK4WeB+YW5fdChQCbc1s5/AEwxYEJyBuizSZqxXeAnPrLEmTga3WvD5NUgKYZGbto0nmaou3wNy6zFJdXGtmZeTYCQdXM17A3Lrs61Tjlkk6Hsjlyz5chvwQ0q2zJLUCXiQ46/gJQaurK8FZySPMbGaE8Vwt8ALm1nmS9iWYQVrAV2b2VsSRXC3xAuaciy3vA3POxZYXMOdcbHkBczlHUoGkc+pw+ydJGljFawZIuqSa213y+5K56vIC5nJRAZCygIUXoToHeAFzuekWYEtJn0u6TVIPSe9IegqYKKkoHEgRAEmXSBoQPt5S0uuSPpH0XjhOWaUk9Zb0kaTPJL0pqUXS6k6S3pY0WdLpSe+5VNI4SRMkXV+7H91Vhw+n43LRFcD2ZtYZQFIPoFu4bIqkojTvHQScZWaTJe0C/AvYN83r3wd2NTOTdBpwGfDXcF1HYFegIfCZpFeA7YH2YR4BwyTtZWbv1uiTut/FC5iLi4/NbEq6F0hqBHQHnpNUsXj9KrbbGhgqqSWwHpC8j/+a2TJgmaR3CIrWHkBPoGJM/UYEBc0LWAS8gLm4WJr0eCWrd39UjLaaB5RUtNwydC9wp5kNC1t6A5LWrXmRpBG0um42swersQ9XR7wPzOWiUqBxmvU/ARtLaiZpfeAQADNbDEyRdAyAAp2q2FdToOKWohPXWHeYpPqSmgE9gHHACOCUsLWHpFaSNs78o7na5C0wl3PMbIGkMWFH/WvAK2usXyHpBuAjgkO+5Buz/wzcH846VA94Bvgize4GEBxyzgQ+BNomrfs43PdmwI1mNguYJWlb4IPwMHUJcDwwt4Yf1/0OfiuRcy62/BDSORdbXsCcc7HlBcw5F1tewJxzseUFzDkXW17AnHOx5QXMORdbXsCcc7H1/zY/m8WqM5cdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_test =  prediction_df['actual']\n",
    "predictions =  prediction_df['pred']\n",
    "\n",
    "cm = confusion_matrix(y_test, predictions)\n",
    "sns.heatmap(cm, square=True, annot=True, cmap='RdBu', cbar=False,\n",
    "xticklabels=sections, yticklabels=sections)\n",
    "plt.xlabel('true label')\n",
    "plt.ylabel('predicted label')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implement Naive Bayes Using Sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 536,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 525,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_df = final_df[['combined_text_cleaned','section_name']].copy()\n",
    "model_df = model_df.reset_index(drop=True)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(model_df['combined_text_cleaned'],\n",
    "                                                    model_df['section_name'], random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 527,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(strip_accents='ascii', token_pattern=u'(?ui)\\\\b\\\\w*[a-z]+\\\\w*\\\\b', lowercase=True, stop_words='english')\n",
    "X_train_cv = cv.fit_transform(X_train)\n",
    "X_test_cv = cv.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 562,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_freq_df = pd.DataFrame(X_train_cv.toarray(), columns=cv.get_feature_names())\n",
    "word_freq_df['class'] = y_train "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 641,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Arts\n",
      "deaths         76\n",
      "notable        74\n",
      "coronavirus    52\n",
      "said           47\n",
      "times          44\n",
      "world          43\n",
      "week           39\n",
      "art            38\n",
      "year           38\n",
      "president      36\n",
      "dtype: int64\n",
      "\n",
      "Business\n",
      "deaths      68\n",
      "notable     65\n",
      "n           47\n",
      "week        43\n",
      "times       42\n",
      "year        41\n",
      "pandemic    38\n",
      "world       36\n",
      "people      35\n",
      "season      35\n",
      "dtype: int64\n",
      "\n",
      "Obituaries\n",
      "deaths         75\n",
      "notable        66\n",
      "times          55\n",
      "n              54\n",
      "world          49\n",
      "week           38\n",
      "s              35\n",
      "coronavirus    35\n",
      "l              34\n",
      "pandemic       34\n",
      "dtype: int64\n",
      "\n",
      "Sports\n",
      "deaths         64\n",
      "notable        60\n",
      "week           35\n",
      "s              35\n",
      "season         29\n",
      "coronavirus    28\n",
      "president      27\n",
      "year           27\n",
      "pandemic       26\n",
      "day            25\n",
      "dtype: int64\n",
      "\n",
      "World\n",
      "amazon           3\n",
      "high             2\n",
      "assails          2\n",
      "virus            2\n",
      "brazil           2\n",
      "toll             2\n",
      "exceptionally    2\n",
      "taking           2\n",
      "pandemic         2\n",
      "region           2\n",
      "dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# word count per section\n",
    "for s in sections:\n",
    "    df = word_freq_df[word_freq_df['class']==s].copy()\n",
    "    df.drop('class',inplace=True,axis=1)\n",
    "    df = df.sum().sort_values(0, ascending=False)\n",
    "    print(s)\n",
    "    print(df.head(10))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 535,
   "metadata": {},
   "outputs": [],
   "source": [
    "naive_bayes = MultinomialNB()\n",
    "naive_bayes.fit(X_train_cv, y_train)\n",
    "predictions = naive_bayes.predict(X_test_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 538,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score:  0.9056\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy score: ', accuracy_score(y_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(91.68, 0.5, 'predicted label')"
      ]
     },
     "execution_count": 540,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATAAAAEwCAYAAAAw+y3zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5wV1fnH8c93C70tRVgBWRRQUYoKqICKRLFHJfb2ExNrjMaSxMRosEUUNdZobGCJCmIXGxZAEJWODQFBpPeyLEjZfX5/zCwusOUCe+/cIc/79drX3il3zjOXy7Nnzpw5R2aGc87FUUbUATjn3I7yBOaciy1PYM652PIE5pyLLU9gzrnY8gTmnIutrKgDiLsrX5kSm34oe154WtQhJOyHgg1Rh7BdamXFpy6Qk50ZdQjb5YaC6SprW3w+deec24onMOdcbHkCc87Flicw51xseQJzzsWWJzDnXGx5AnPOxZYnMOdcbHkCc87Flicw51xseQJzzsWWJzDnXGx5AnPOxZYnMOdcbHkCc87Flicw51xs+YCGaaBHqwZ0zWuABKNnLWf4jKU0rVuNsw5oSnZmBkVmDJo4j9kr1kUd6hZyWrfkhIH3b16um9ecz+54gIn/fibCqEp33lN3s/8JPclfvIw72h8TdTgVyqpahcs+HkRm1SpkZmXy1avvMezW+yt+Y0Sq1q3NcY/8k0ZtW2MG71x+A/O/nJT0cnfpBCbpVOBVYF8zm1rK9nrAOWb275QHF8qtU5WueQ3o/8l0CouMK7q35JuFqzmlXS7vfreYbxfl07ZJbU5pl8sDI2dGFWapVkyfxfPdTgZAGRlcMu1TZrw1LOKoSvf5wCGMePgZLnjmvqhDScim9Rt4vNe5bChYS0ZWFpcPH8z37w3npxQkhR1xVP+/M3PYSF4/7w9kZGeTXaNaSsrd1S8hzwZGAWdtvUFSJlAPuCLVQZXUpHY1fly+lo2FRpHBjCUFdNi9LgDVsoN/nupZmaxatzHKMCu0R49DWTnrJ/LnzI86lFLN+PRLCpavijqM7bKhYC0AmdlZZGZnYZaeo5dXqV2L5t06M+WZlwEo2riR9avyU1L2LpvAJNUCugG/JUxgknpI+kTSC8BXQD9gL0mTJPWXlCtpZLj8taTDkh3n/NU/06phTWpWySQ7U+zXpDY51bMZMnk+p7TL5bbj9uHU9rm88c3CZIeyU/Y+7QS+f3lo1GHsUpSRwdVj3+ameWOZ/tFo5oydHHVIparXsjlrly7nhP/cRZ/P3uC4R+4gu0b1lJS9yyYw4BTgPTObBiyXdGC4vgtwo5m1BW4AfjCzjmb2J+Ac4H0z6wh0AEqtr0u6RNI4SeO+GTZkp4JclL+eYdMWc2X3Pfl9t5bMW/UzhWYctmcDXp08n5vencork+dz7kHNdqqcZMrIzmav43/FtNfejTqUXYoVFfFA5xP5Z8uuNO/Unsb7tYk6pFJlZGbSpON+THjiBQZ0PZmNa9dxyHWXpqbslJQSjbOBl8LXL4XLAF+a2awy3jMW6COpL9DOzEqtB5vZ42bWycw67Xf0zs/0M+bHFdz18XTuHzmTgg2FLFmzgYNb5DBp/moAJs5bRYucGjtdTrK07HU4iyZ9w9oly6IOZZf086p8Zo78gr17HR51KKXKn7+Q/HkLWTAuqCFOfe09GnfcLyVl75IJTFIDoCfwpKQfgT8BZwICCsp6n5mNBA4H5gHPSbog+dFCrarBNFc51bPp0LQO4+asZNW6jbRuWBOANo1qsWTN+lSEskP2Pu1Evh/ydtRh7FJqNqxPtbq1AciqVpVWPbux+Pv0uolTrGDRUlbPXUD91i0ByOtxKMumzkhJ2bvqXcjTgGfNbHM9VtIIoPtW++UDtUvs0wKYZ2ZPSKoJHAg8m+xgf3dIHjWrZFJYZAyeOJ91Gwt5YcJcTuuwOxkSm4qMFyfMS3YYOySrejVa9OzKh1ffFHUo5erz3wdp3eMQajXM4fafxjC0778Y8/TgqMMqU+3c3Tjjqf5kZGaiDDFlyDtMfefjqMMq07Drb+Okp+8ls0o2K2fNYehlN6SkXKXrnY2dIWk40M/M3iux7irgcoI2rxNLrH8BaA+8C3xNUFvbCKwBLijnchPwiW2TxSe2TZ5daWLbXbIGZmY9Sln3IPBgKevP2WpV+vXCdM6VKj5/NpxzbiuewJxzseUJzDkXW57AnHOx5QnMORdbnsCcc7HlCcw5F1uewJxzseUJzDkXW57AnHOx5QnMORdbnsCcc7HlCcw5F1uewJxzseUJzDkXW57AnHOxtUsOaJhKu53XO+oQEvbQ+f2iDiFhxzxxfdQhbJcNRbEZmJc5aT7H6PbwGphzLrY8gTnnYssTmHMutjyBOediyxOYcy62PIE552LLE5hzLrY8gTnnYssTmHMutjyBOediyxOYcy62PIE552LLE5hzLrY8gTnnYssTmHMutjyBOediywc0TDNXTR3O+vwCrLCQok2FPNn91Eov467/O4ie7XJZlr+eY28Zts32k7s057Jj9wagYH0hN/13At/NXbVTZVbJyuDePp3Zv0UOKws2cOXjnzNv2Vr2bVaX2889kFrVsygqMh5+ZypDx83dqbLK0vaYIzjjgZvJyMxk9JODeP+uR5NSTmWo1yyXcwbeS53GjbCiIsY8+SIjHxoYdVilOu+pu9n/hJ7kL17GHe2PSWnZkdTAJBVKmiRpsqQJkrru4HEuk3RBZccXtWePPY/HD/l1UpIXwCufzebCB0eVuX3O0rWcec8Ijrv1Qx4a+h3/PP+ghI/dtEENXrzuiG3Wn9Etj1VrN3Dk39/jqQ+ncUPvdgD8vKGQ6waM5Zi+w/i/B0Zx85kdqF09e/tPqgLKyODsR27l4eMu5Ja2R9P57F+Tu2+rSi+nshRt2sSbf7qDfu2O5v5uvel2+QU0TtN4Px84hEeO+79Iyo6qBrbOzDoCSDoGuBPY9ltfATN7rLID+1/w5fSlNG1Qo8ztE2Yu2/x64sxlNKlXffPyKQfvwYU9W5GdlcGkWcu56b8TSGQ05aM77s79b30LwLvj53HL2QcAMGvxms37LF71M8tWr6dB7arbe0oVyuvSkcUzZrN01hwAxr70Fu1P7sWC72ZUelmVYfXCJaxeuASA9WsKWDR1BnWbNmFRGsY749Mvqd+iWSRlp0MbWB1gBYCkHpLeLt4g6WFJF4av+0n6VtIUSfeE6/pKuj58PVzSXZK+lDRN0mHh+kxJ/SWNDd97abg+V9LIsCb4taTDwn0HhstfSbomtR8FmBnnvTWQ341+nQMvOjPVxW/jzG4tGfH1QgD2alKbEzs147S7P+GE2z6ksMg45eA9EjpO43rVWbB8HQCFRUb+uo3k1KqyxT4d8nLIzspg9pI1pR1ip+Q0bcyKOfM3L6+cu4Ccpo0rvZxkyGnRlGYd2zL7i0lRh5J2oqqBVZc0CagG5AI9y9tZUn3gVGAfMzNJ9crYNcvMukg6HvgHcBTwW2CVmXWWVBUYLekDoDfwvpndISkTqAF0BJqa2f5huWWVkzQDep7JmgWLqdGoPue9/QxLv5/JT6PHpjoMAA7ZuxFndM/j9LuHA9Bt393Yv0UOb/ztVwBUy85kWf56AB67/FCaN6xJdmYGu9evwdCbjgrO56PpDPlsNtK2x7cSNbdGdatx30VduG7A2C3WV5pSArCkFFS5qtSsQZ/Bj/LatbexPr/yE3vcpcMl5KHAs5L2L2f/1cDPwJOShgJvl7Hfq+Hv8UBe+LoX0F7SaeFyXaA1MBZ4WlI28LqZTZI0E9hT0kPAUOCD0gqRdAlwCcBJWY3olFWnovNN2JoFiwFYu2Q53785jKad20eSwPZpWpd+FxxEnwdGsbJgAwACXhkzm/6vfb3N/pc9OgYI2sDuubAzZ987YovtC1esI7d+dRauXEdmhqhdPXvzcWtVy+LpP3Tj3je+ZtKs5Uk5nxVzF5LTfPfNy/Wa5bJy/uKklFVZMrKy6PPyo4x/8Q2+ev39qMNJS5FfQprZGKAh0AjYxJYxVQv32QR0AV4BTgHeK+Nw68PfhfySnAX8wcw6hj8tzewDMxsJHA7MA56TdIGZrQA6AMOB3wNPlhHz42bWycw6VWbyyq5RnSq1am5+vedR3Vn8zfRKO36idq9fnUcvP5Rrnxq7RRvV6KmLOe7AppvbqOrWyKZp/bLb0kr6cPICfnNoCwCOO6gpY6YGySM7Uzx2eVdeHTObd8bPq+Qz+cXssZPZrXUeDfKakZmdTeezTmLKm9vegU0nZz1xF4u+m8GI+5+KOpS0FXk3Ckn7AJnAMmA20Da81KsG/AoYJakWUMPM3pH0ObA9LZnvA5dL+tjMNkpqQ5C0GgLzzOwJSTWBAyW9A2wws1ck/QAMrKzzTETN3RpyxqB/A8Ff368HvckPw0ZWejkP/K4Lh+zdiJxaVfnsruO5/81vycoM/m68MHImV53QlpyaVbjt3KChfVNhESf/82NmLMjn3je+4dk/HkaGYGOhcfMLE5m3fG2FZQ4aNYt//bYLn9x+LKsKNvCHJ74A4IROzenSpiE5tapwWtc8AK4fUPk1zqLCQgZdeTNXvf8sGZmZfPb0YBZ8m/o/Dolq2a0Tnc/vzfwpU7l+3FAAht7Un+/eHR5tYKXo898Had3jEGo1zOH2n8YwtO+/GPP04JSUrSjaASQVAl8VLwJ/M7Oh4ba7gZOB6cAG4E2CJPQGQVITcI+ZPSOpL7DGzO6RNBy43szGSWoIjDOzPEkZwO3ASeF7lxDU4k4B/gRsBNYAFxDcUBjAL7XAv5rZu+Wdy63VW6V/Q0pogE9smzTVM0tp5EtTcZqEF+CRoh/L/HAjSWC7Ek9gyeEJLHl2pQQWeRuYc87tKE9gzrnY8gTmnIutMu9CSvoKKO1iWYCZWfukReWccwkorxvFiSmLwjnndkCZCczMZhe/ltQCaG1mH0qqXt77nHMuVSpsA5N0MTAE+E+4qhnwejKDcs65RCTSiP97oBvB84iY2XRgt2QG5ZxziUgkga03sw3FC5KyKL1x3znnUiqRBDZC0t8IhsA5GngZeCu5YTnnXMUSSWA3EDw/+BVwKfAO8PdkBuWcc4mo8G6imRVJegb4guDS8XvzByidc2mgwgQm6QTgMeAHgk6sLSVdWtEoDc45l2yJ9Oe6FzjSzGYASNqLYLRST2DOuUgl0ga2uDh5hWYC6T0Wr3Puf0J5z0L2Dl9+E45UOpigDex0gvHknXMuUuVdQp5U4vUifpm3cQmQk7SIYqagsCjqEBJ2/JPxGSTwiY7lTlSVdq75ZnjUIfxPKu9ZyD6pDMQ557ZXInchqxHMrbgf4SxBAGZ2URLjcs65CiXSiP8c0AQ4BhhB8DB3fjKDcs65RCSSwFqZ2U1AgZk9A5wAtEtuWM45V7FEEtjG8PfKcPbsuvwy67VzzkUmkY6sj0vKAW4imKOxFnBzUqNyzrkEJPIs5JPhyxHAnskNxznnEldeR9Zry3ujmd1X+eE451ziyquB1U5ZFM45twPK68h6SyoDcc657eUT2zrnYssTmHMutjyBOediy+9COudiK5G7kHsDnQk6sUIwzM7IZAblnHOJqPAupKQPgAPNLD9c7kswtZpzzkUqkTawPYANJZY34M9COufSQCLPQj4HfCnpNYIhpU8Fnk1qVP+jsqpW4bKPB5FZtQqZWZl89ep7DLv1/qjDKtN5T93N/if0JH/xMu5of0xSynj8H304/vAOLFm+mgNO3/YR3JN6dKTv5adSZMamwiKu6/8in02avlNlVsnOYsBtv+OAfVuwfFUB5/7lUWYvWEaHNs156MbzqVOzOoWFRfR76m1e/iA5o6vH6btQr1ku5wy8lzqNG2FFRYx58kVGPjQwJWUrkSkeJR0IHBYujjSziQkdXGoGPAK0JajtvQ38CTgH6GRmV5byns/MrKukPKCrmb2QSFkJxnMrQfwfVtYx/1Jlz0qdI7NKzRpsKFhLRlYWlw8fzFvX3spPX06qlGOv2VS5w1+3OqwL69cUcMEz91V6AnuiQzCkdPcD27Bm7c9BQiklgdWsXpWCdesBaNe6GS/cdTntet+YUBktchvw5K2/5eiL795i/aWnH0m7Ns248o7nOOOYLpx85IGce8NjtN6jMYYx46fF5Daqx+f/vZn2vW9k1Zp1SRlSOlnfhQ1FlTuta50mjaiTuxtzJ35D1Vo1ufbLt3j6N5ew6LsZFb85Af/aNEtlbUu0G0UNYLWZPQDMldSyojdIEvAq8LqZtQbaEIxkcUd57zOzruHLPIJEVykkZZrZzZWZvJJhQ8FaADKzs8jMziKd5xCe8emXFCxfldQyRk2YxopVBWVuL05eADWqV93i8zrn+EMY/dzfGftSXx658QIyMsr8f7CFk3ocwHNvfQbAKx+O48gu+wIw/adFzPgpmJBrwZKVLFmRT6P6yXviLi7fhdULlzB34jcArF9TwKKpM6jbtElKyq4wgUn6B/AX4K/hqmzg+QSO3RP42cwGAJhZIXANcBFBQmwu6T1J34dlFJe3JnzZDzhM0iRJ10i6UNLDJfZ7W1KP8PWjksZJ+kbSLSX2+VHSzZJGAadLGijptHDbQZJGSBov6X1JueH6qyR9K2mKpJcSOM9KpYwMrh77NjfNG8v0j0YzZ+zkVIcQOycfeSBfvXoHbzx4NRffMgCAfVrmcnqvLhzR5046n9WXwqIizjn+0ISO13S3esxduByAwsIiVq1ZR4N6tbbYp9N+LamSlckPc5ZU7smUEMfvQk6LpjTr2JbZX1TOVUNFEmkDOxU4AJgAYGbzJSXyZ2c/YHzJFWa2WtJPYbldgP2BtcBYSUPNbFyJ3W8ArjezEwEkXVhOWTea2XJJmcBHktqb2ZRw289m1j08xrHh72zgIeBkM1si6UyCmuFFYbktzWy9pHqlFSbpEuASgF6ZDeiYUSeBjyMxVlTEA51PpFrd2lzw8mM03q8Ni76ZVmnH3xW98ckE3vhkAt0PbEPfK07luMvu4cgu+3JA2zzGPH8TANWrVmHJ8mAk9JfvvZK8pg2pkp1F8yb1GftSXwAeeuFDnn1zFMHFw5ZK1n6aNKzLwNsv5qKbn0xqrShu34UqNWvQZ/CjvHbtbazPX1PxGypBIglsg5mZJAOQVDPBY4ug0b+s9cPMbFl4zFeB7sC4UvZPxBlhUskCcgna3IoT2KBS9t+bIHkOC7+smcCCcNsU4L+SXgdeL60wM3sceBwqvw2s2M+r8pk58gv27nV4Wn9p08moCdPYs1kjGtSrhSSef2s0f3/olW32O/26oCJfVhvY3EUraNakPvMWryAzM4O6taqzPLyMrV2zGm88+Ef+8cirfPnVzOSfFPH4LmRkZdHn5UcZ/+IbfPX6+6krN4F9Bkv6D1BP0sXAh8CTFbwH4BugU8kVkuoAzYFCtk1uFSWCTVvFWy08ZkvgeuBXZtYeGEqJ2ZOA0hpQBHxjZh3Dn3Zm1ivcdgLBjYeDgPGSEknylaJmw/pUqxtUbrOqVaVVz24s/j41/0niaq/mu21+3XGfPaiSncWylWv45MvvOPWoTjTKCT7PnDo12SO3QULHfHvEJM4/KWiK/c1RnRg+dioA2VmZvHzvlTz/9me88uGO/q1NTNy+C2c9cReLvpvBiPufSmm5iYzIeo+ko4HVBDWXm81sWALH/gjoJ+kCM3s2vLy7FxhIcNl4tKT6wDrgFILLt5Ly2XJMsh+BKyRlAE0JLkEB6hAkqVWSGgPHAcMriO17oJGkQ81sTHhJ2Qb4DmhuZp+E7WbnENx4WJnA+e602rm7ccZT/cnIzEQZYsqQd5j6zsepKHqH9Pnvg7TucQi1GuZw+09jGNr3X4x5enCllvHcnZdy+EF707BeLWa+dw+3PvYG2VmZADwxZDin/uogzjuxKxs3FbJu/QbO/ctjAHw3cz59H3mVdx69jgyJjZsKuarf8/y0YFmFZQ54fSQDb7+Yb9+4kxWrCzjvhv8AcHqvzhx2YBsa1KvFBb/uBsDvbn6KydPmVOo5Q7y+Cy27daLz+b2ZP2Uq148bCsDQm/rz3bvDk152hd0oJN1lZn+paF0Z720O/BvYh6D29A5Bbels4HigJtAKeKFEz/81ZlYrTCrvAQ0Jkt79BDcPOgJfA42BvmY2XNJA4GBgJrAeeNPMBkr6kaC7xtLw2AOBt81siKSOwIMEk5RkhccfCHwSrhPwvJn1K+8ck3UJmQyV3Y0imYq7UcRFnGbmruxuFMlWXjeKRBLYBDM7cKt1U8LLtf95nsCSwxNY8uxKCay80SguB64A9pI0pcSm2sBnlReec87tmPLawF4A3gXuJOhaUCzfzJYnNSrnnEtAmXchzWyVmf0IPAAsN7PZZjYb2Cjp4FQF6JxzZUmkG8WjQMleaQXhOueci1QiCUxWoqXfzIpIrAOsc84lVSIJbGb4fGB2+HM1QXcF55yLVCIJ7DKgKzAPmEvQ3+qSZAblnHOJSKQn/mLgrBTE4pxz26W8fmB/NrO7JT1EKc8pmtlVSY3MOecqUF4N7Lvwd3KfWnXOuR1U3qxEb4W/n0ldOM45l7jyLiHfopwhbszs10mJyDnnElTeJeQ94e/eQBN+GUb6bIKhbZxzLlLlXUKOAJB0m5kdXmLTW5J8Zm7nXOQS6QfWSNKexQvhCKiNkheSc84lJpFHgq4Bhksq7n2fB1yatIiccy5BiU5sW5VgVFWAqWa2vrz9/5dcprzYjA5XPTOxeRHTwbrC2HysALx11t+iDiFhZ7x8Z9QhbJedmthWUg2C2bSvNLPJwB6STqzE+Jxzbock0gY2ANgAFM8KOhe4PWkROedcghJJYHuZ2d3ARgAzW0cw4YVzzkUqkQS2QVJ1wk6tkvYimPnHOecilchdyH8QTG/WXNJ/gW7AhckMyjnnElFuAgsnkc0h6I1/CMGl49XF8yw651yUyk1gZlYk6UozGwwMTVFMzjmXkETawIZJul5Sc0n1i3+SHplzzlUgkTawi8Lfvy+xzoA9S9nXOedSJpEhpVumIhDnnNteFSYwSdWAK4DuBDWvT4HHzOznJMfmnHPlSuQS8lkgH3goXD4beA44PVlBOedcIhJJYHubWYcSy59ImpysgJxzLlGJ3IWcKOmQ4gVJBwOjkxeSc84lJpEa2MHABZJ+Cpf3AL6T9BVgZtY+adE551w5EklgxyY9Cuec2wGJdKOYnYpAXKDtMUdwxgM3k5GZyegnB/H+XY9GHVKZ6jXL5ZyB91KncSOsqIgxT77IyIcGRh1WmVLx2d576REcdcAeLF29jl/9ecg220/t1oorfh00Ka/9eSN/fWoU3/60fKfKrJKVwQNXHEm7lg1ZsWY9lz/wIXOXrmG/Fg2486Lu1KqRTWGR8dBrE3nz85kVH3A7Rfk9SKQNLG1IulHSN5KmSJoUtsft7DF7SOpaGfHtLGVkcPYjt/LwcRdyS9uj6Xz2r8ndt1XUYZWpaNMm3vzTHfRrdzT3d+tNt8svoHGaxpuqz3bwiO85t987ZW6fszif0259i6P/8gr3vzqRuy4+vMx9t9asYS1evmnbsUTPPnIfVhWsp/s1g3jina+48Zzgv8W69Zu4+tFP6PmnIZzX7136XtCVOjWqbP9JVSDK70FsEpikQ4ETgQPDdrejgDk7ecwsoAeQFgksr0tHFs+YzdJZcyjcuJGxL71F+5N7RR1WmVYvXMLcid8AsH5NAYumzqBu0yYRR1W6VH22X0xdyMo1ZY82NW76IlYVbABgwoxF5NavuXlb7+6tePu2U/jgzt7c9dvDyFBiw+71OqgFL4+cBsDQL2bSff+mAMxcuIpZC1cDsGjFWpatXkeDOtV26LzKE+X3IDYJDMgFlhaPx29mS81svqQfJd0l6cvwpxWApBaSPgprax9J2iNcP1DSfZI+AQYBlwHXhDW6wySdLulrSZNTPX1cTtPGrJgzf/PyyrkLyGnaOJUh7LCcFk1p1rEts7+YFHUopUrHz/asHvvwyaTgb3Cr3evx60P24pS+b9Drr69SaEX07p5YLaZJ/ZrMX1YAQGGRsXrtBnJqV91in457NSI7K4MfF62u3JPYSqq/B4k04qeLD4CbJU0DPgQGFc9dCaw2sy6SLgDuJ6ipPQw8a2bPSLoIeBA4Jdy/DXCUmRVK6gusMbN7AMK7q8eY2TxJ9VJ2dkHh26xKZNKVqFWpWYM+gx/ltWtvY33+mqjDKV2afbZd2+Zy9pF7c2rfNwHovn9T2u3ZkHduPxWAalWyWLoqeNjlyWuPZo9GtcnOyqRpw1p8cGfvYP17XzN4xLTSTi0cfjSwW73qPHjFkfzx0eEk85Sj+B7EJoGZ2RpJBwGHAUcCgyTdEG5+scTvf4WvDyUYxwyCJwfuLnG4l82ssIyiRgMDJQ0GXi1tB0mXAJcAHEZ92lJ7B85oWyvmLiSn+e6bl+s1y2Xl/MWVcuxkycjKos/LjzL+xTf46vX3ow6nTOn02e67R336X3IE5/d7lxXh5aYEL4+cRr+Xxm6z/+/uGwYEbWD/urwHp9/29hbbFywrYPcGNVmwvIDMDFGnRpXNx61VPZtn/3wcdw8ey4QZyTvfqL4HcbqExMwKzWy4mf0DuBL4TfGmkruV9fYSrwvKKeMy4O9Ac2CSpAal7PO4mXUys06VlbwAZo+dzG6t82iQ14zM7Gw6n3USU94cVmnHT4aznriLRd/NYMT9T0UdSrnS5bPdvUFNnrjmaK5+5BNmLly1ef2or+dxYpc9N7dR1atZlaYNayV0zA/Gz+b0w9sAcMLBezL6m3kAZGdm8NS1vRjy6TTe/mJWJZ/JlqL6HsSmBiZpb6DIzKaHqzoCs4F2wJlAv/D3mHD7Z8BZBLWvc4FRZRw6H6hTopy9zOwL4AtJJxEksmWVezalKyosZNCVN3PV+8+SkZnJZ08PZsG30yt+Y0RadutE5/N7M3/KVK4fF4x3OfSm/nz37vBoAytFqj7bR/7Qk0P33Z36tasx7uFzuGfIeLKzgnrCcx9+xzW9DyKnVjX+eVE3ADYVGcff+BrT563k7sFjefGvx6MMsWlTETcOGM28pRVfir00/HsevOJIRv3rTFauWc8VD30EwEmH7vCTRLwAABR0SURBVMnB++SSU6sqZ4QJ7prHRpR3qB0S5fcgoYlt00F4+fgQUA/YBMwguIwbRzD12/EENcqzzWyGpDzgaaAhsAToY2Y/SRoIvG1mQ8LjtgGGAEXAHwhmIm9NMHz2R8AfrZwPySe2TQ6f2DZ5dqWJbWNTAzOz8ZTS3UFBC+YjZnbLVvv/CPQs5TgXbrU8DSj5ONSnOx+tcy4VYtUG5pxzJcWmBlYWM8uLOgbnXDS8Buaciy1PYM652PIE5pyLLU9gzrnY8gTmnIstT2DOudjyBOaciy1PYM652PIE5pyLLU9gzrnY8gTmnIstT2DOudjyBOaciy1PYM652Ir9cDpRy6uRHXUICWtVq/InNU2WDxeXOW1BWjrvlX5Rh5CwgkFvRB1CpfEamHMutjyBOediyxOYcy62PIE552LLE5hzLrY8gTnnYssTmHMutjyBOediyxOYcy62PIE552LLE5hzLrY8gTnnYssTmHMutjyBOediyxOYcy62PIE552LLE5hzLrZ8RNY0U7VubY575J80atsaM3jn8huY/+WkqMPa7KD7byP36CNYv3Q5w444BYB2N19Hbq8eFG3cSMGPcxh39d/ZuDo/4ki31faYIzjjgZvJyMxk9JODeP+uR6MOqUxZVatw2ceDyKxahcysTL569T2G3Xp/1GFtoUerBnTNa4AEo2ctZ/iMpTStW42zDmhKdmYGRWYMmjiP2SvWJS2GWNbAJP1L0h9LLL8v6ckSy/dKujbBYw2UdFop63tIertyIk7cUf3/zsxhI3niwGN5+pCTWPb9D6kOoVyzX3qdUWddusW6xSPGMOyIU/jwyN6s+WE2+1x1cUTRlU0ZGZz9yK08fNyF3NL2aDqf/Wty920VdVhl2rR+A4/3OpcHOp3A/Z1OpE2vw9mjS8eow9ost05VuuY1oP8n07nzw2nsn1ubRrWqcEq7XN79bjH9PprO298u4pR2uUmNI5YJDPgM6AogKQNoCOxXYntXYHRFB5GUmZTodlCV2rVo3q0zU555GYCijRtZvyq9ajJLPx/PhpWrtli3aMRnWGEhAMvGT6b67o2jCK1ceV06snjGbJbOmkPhxo2Mfekt2p/cK+qwyrWhYC0AmdlZZGZnYWYRR/SLJrWr8ePytWwsNIoMZiwpoMPudQGolh2klepZmaxatzGpccQ1gY0mTGAEietrIF9SjqSqwL5APUkTJX0l6elwPZJ+lHSzpFHA6SUPKulYSVPDbb1TdzqBei2bs3bpck74z130+ewNjnvkDrJrVE91GDsl75zeLPzo06jD2EZO08asmDN/8/LKuQvIaZp+ibYkZWRw9di3uWneWKZ/NJo5YydHHdJm81f/TKuGNalZJZPsTLFfk9rkVM9myOT5nNIul9uO24dT2+fyxjcLkxpHLBOYmc0HNknagyCRjQG+AA4FOgHTgCeBM82sHUFb3+UlDvGzmXU3s5eKV0iqBjwBnAQcBjQpq3xJl0gaJ2ncl5tWlbXbdsvIzKRJx/2Y8MQLDOh6MhvXruOQ6y6t+I1pYp8/XoJt2sRPr6T8yrti0jar0qlGUxorKuKBzifyz5Zdad6pPY33axN1SJstyl/PsGmLubL7nvy+W0vmrfqZQjMO27MBr06ez03vTuWVyfM596BmSY0jlgksVFwLK05gY0oszwNmmdm0cN9ngMNLvHdQKcfbJ3zPdAu+2c+XVbCZPW5mncysU5esujt/JqH8+QvJn7eQBeOCv7RTX3uPxh33q+Bd6aHFGSeTe/QRfHnFX6IOpVQr5i4kp/num5frNctl5fzFEUaUuJ9X5TNz5Bfs3evwindOoTE/ruCuj6dz/8iZFGwoZMmaDRzcIodJ81cDMHHeKlrk1EhqDHFOYMXtYO0ILiE/J6iBdQUmVPDesiYdjPRPcsGipayeu4D6rVsCkNfjUJZNnRFlSAlpfGR39r7yt4y+4EoK1/0cdTilmj12Mru1zqNBXjMys7PpfNZJTHlzWNRhlalmw/pUq1sbgKxqVWnVsxuLv58ZcVRbqlU1aELOqZ5Nh6Z1GDdnJavWbaR1w5oAtGlUiyVr1ic1hjh3oxgNXAfMNLNCYLmkegRtYn8ALpbUysxmAOcDIyo43lSgpaS9zOwH4Owkxl6mYdffxklP30tmlWxWzprD0MtuiCKMMnV5rD+Nunamav16HD/xI77t/wj7XHUxGVWyOXxwcCN42fjJTPzzrRFHuqWiwkIGXXkzV73/LBmZmXz29GAWfDs96rDKVDt3N854qj8ZmZkoQ0wZ8g5T3/k46rC28LtD8qhZJZPCImPwxPms21jICxPmclqH3cmQ2FRkvDhhXlJjULq3A5QlvIO4AnjQzP4erhsIHGpme0v6FXAPQZIeC1xuZusl/Qh0MrOlJd7ztpkNkXQscD+wFBgF7G9mJ5YXR7+arWPzAfrM3MlTNzs+FzMFL74edQjb5eHftN+2ATMU2xpYWOuqs9W6C0u8/gg4oJT35ZXznvcI2sKcczEQnz8bzjm3FU9gzrnY8gTmnIstT2DOudjyBOaciy1PYM652PIE5pyLLU9gzrnY8gTmnIstT2DOudjyBOaciy1PYM652PIE5pyLLU9gzrnY8gTmnIstT2DOudiK7YisuzpJl5jZ41HHkYg4xQrxijdOsULq4/UaWPq6JOoAtkOcYoV4xRunWCHF8XoCc87Flicw51xseQJLX7Fp9yBesUK84o1TrJDieL0R3zkXW14Dc87Flicw51xseQJzzsWWJzD3P0VShqQ6Fe+ZHiTlSGofdRzpyhNYmpDUTVLN8PV5ku6T1CLquMoi6WpJdRR4StIESb2ijqs0kl4IY60JfAt8L+lPUcdVFknDw3jrA5OBAZLuizqukiS9JenNsn5SFYcnsPTxKLBWUgfgz8Bs4NloQyrXRWa2GugFNAL6AP2iDalMbcNYTwHeAfYAzo82pHLVDePtDQwws4OAoyKOaWv3APcCs4B1wBPhzxrg61QFkZWqglyFNpmZSToZeMDMnpL0f1EHVQ6Fv48n+E82WZLKe0OEsiVlEySwh81so6R07j+UJSkXOAO4MepgSmNmIwAk3WZmh5fY9JakkamKw2tg6SNf0l+B84ChkjKB7IhjKs94SR8QJLD3JdUGiiKOqSz/AX4EagIjw0vz1ZFGVL5bgPeBGWY2VtKewPSIYypLozA+ACS1JKiRp4R3ZE0TkpoA5wBjzexTSXsAPcwsLS8jJWUAHYGZZrYybK9pZmZTIg4tIZKyzGxT1HGURlI3Mxtd0bp0IOlYgt73M8NVecClZvZ+Ksr3S8j0cY2Z/aV4wcx+krRflAFV4FBgkpkVSDoPOBB4IOKYSiWpMfBPYHczO05SW4L4n4o2sjI9RPB5VrQucmb2nqTWwD7hqqlmtj5V5XsNLE1ImmBmB261boqZpeUtdElTgA5Ae+A5gmTQ28yOiDSwUkh6FxgA3GhmHSRlARPNrF3EoW1B0qFAV+CPwL9KbKoDnGpmHSIJrBSSepe33cxeTUUcXgOLmKTLgSuAvcKkUKw2kHaXDCXE6aZDQzMbHLYxYmabJBVGHVQpqgC1CP5f1i6xfjVwWiQRle2kcrYZ4Ansf8QLwLvAncANJdbnm9nyaEJKSPFNh/OBw9L8pkOBpAYE/7GQdAiwKtqQtmVmIySNAtqZ2S1Rx1MeM+sTtoOeZmaDo4rDE1jEzGyVpHyCL+3sqOPZDmcS3HS4yMwWhjcd+kccU1muBd4kqOWOJrhLlm41GgDMrDC8IZL2zKxI0pVAZAnM28DShKT/An81s5+ijiVRYXeE1mb2oaQaQKaZ5UcdV2nCdq+9CfqvfW9mGyMOqUyS7gVaAy8DBcXrU9WutD0k3UTQkXUQW8aakqsHT2BpQtLHQGfgS375IpiZnRxdVGWTdDHB+Of1zWyv8E7UY2b2q4hD20xSTzP7uKwG53RMCACSBpSy2szsopQHUwFJs0pZbWa2ZynrK51fQqaPkm0eAroDZ0cUSyJ+D3QBvgAws+mSdos2pG0cAXxM6Q3OKWto3l5m1ifqGBJlZi2jLN8TWJoIG3A7ErQrnUHwjNlj0UZVrvVmtqH46aHwEi2tqvNm9o+wofndKBuat5ekZgT9vroRfKajgKvNbG6kgZUifETrcqD4caLhwH9SdYnujxJFTFIbSTdL+g54GJhDcGl/pJk9FHF45Rkh6W9AdUlHE7TXvBVxTNswsyLgyqjj2E4DCG467A40JfhcS7usTAePAgcB/w5/DgrXpYS3gUVMUhHwKfBbM5sRrpuZqjaEHRXWbH5LMBqFCJ7de9LS8AsVdUPz9pI0ycw6VrQuHUiavHUH29LWJYtfQkbvN8BZwCeS3gNe4peRHtJWWLMpHkIl3RU3fv++xDoD0vWPxNLw8awXw+WzgWURxlOeQkl7mdkPAOGD3SnrJOw1sDQRDrZ3CsGXtSfwDPCamX0QaWBlkNQN6Au0IPhDKFJ492lXFvape5jgeU0Insi4Op36CUr6I0FcOQR/xIrvRuYR9A38OCVxeAJLP2FHxtOBM82sZ9TxlEbSVOAaYDwl/uKaWVrWFCTtD7QFqhWvS9eRPuJA0j0Ez23uC0wD5hF8FwaY2fyUxeEJzO0ISV+Y2cFRx5EISf8AehAksHeA44BRZpaWvfHDy7AHgEMILnXHEIxWMrPcN0ZAUhWgE0EyOzT8WWlmbVNRvt+FdDvqE0n9JR0q6cDin6iDKsNpwK+AhWEfqw5A1WhDKtcLBI/n5BLciXyZX9rD0k11gtEy6oY/8wn7BqaCN+K7HVVc++pUYp0RtN+lm3Xhc3ubFMxItJj0bcCH4MrouRLLz4fPHKYNSY8D+wH5BAnrM+A+M1uRyjg8gbkdYmZHRh3DdhgnqR5BY/N4goknvow2pHJ9IukGgjvSRvDg/NDih7zTpPvHHgS12OkE7V9zgZWpDsLbwNx2kXSemT0v6drStptZWk3/tTVJeUCddB76usTzhcX/OUt2q0mbO73hJC77EbR/dQX2B5YDY8zsH6mIwWtgbnvVDH/XLnevNCLp8NLWmVnKZs9JhKTOwJzi5wvDASJ/QzAhSd80qXltFnZa/lrSSoLx1VYBJxI8I5uSBOY1MLfLk1TyEadqBP/BxqdbFxVJE4CjzGx5mHRfAv5AMHnKvul011TSVQS1rm7ARoI+YWPC31+FHZ2TzmtgbodIuhu4neARnfcI7uz90cyejzSwUpjZFqNRSGoO3B1ROOXJLFHLOhN43MxeAV6RNCnCuEqTBwwh6N6xIKogvBuF21G9LJg9+kSCBtw2wJ+iDSlhcwnaa9JNZjiqBwTdPkr2Zk+ryoaZXWtmQ6JMXpBmH4qLleLx748HXgwve6KMp0ySHuKXBvHi+SwnRxdRmV4kGOVjKUHN9lMASa1IwzH804EnMLej3gofJ1oHXCGpEfBzxDGVZVyJ15sIEm7azfhkZndI+oigA+sHJUb2yCBoC3Nb8UZ8t8Mk5QCrw4koahB0T1gYdVxbk3S1mT1Q0ToXP57A3A6RdEFp69PxAekyJg2eaGYHRBWTqxx+Cel2VOcSr6sRNDpPANImgUk6m2CI7paS3iyxqTbpO76W2w6ewNwOMbMt2mQk1QWeK2P3qHwGLAAaAveWWJ8PpG1PfJc4v4R0lSKc3GGKme0bdSzuf4fXwNwOCXu3l7xL1pYIZ2gujaRRZtY9nPm85F/q4tFj60QUmqskXgNzO0TSESUWNwGz03HaL7dr8wTmdpqkhsCydJyRqFg42GJ3wnkWzWxixCG5SuCPErntIukQScMlvSrpAElfA18DiyQdG3V8pZF0M8EkKQ0IGvQHSvp7tFG5yuA1MLddJI0D/kYwfPDjwHFm9rmkfQh6uKdd36pw0uADzOzncLk6MMFvOMSf18Dc9soysw/M7GWCMeY/BzCzqRHHVZ4fKTEbEcFIoj9EE4qrTH4X0m2vkuM8rdtqW1pV50s8xL0e+EbSsHDTUcCoyAJzlcYvId12kVQIFBB0RagOrC3eBFQzs+yy3ptq4YimEMSZTZB8CwkTr5k9E1ForpJ4AnO7rLBz7R3ARcBsgiaT5sAA4G9mtjHC8Fwl8DYwtyu7G8gBWprZQeENhj0JbkD0jzQyVym8BuZ2WZKmA2227p8mKROYamato4nMVRavgbldmZXWudbMCkmzGw5ux3gCc7uyb0sbt0zSeUA6d/twCfJLSLfLktQUeJXgruN4glpXZ4K7kqea2bwIw3OVwBOY2+VJ6kkwg7SAb8zso4hDcpXEE5hzLra8Dcw5F1uewJxzseUJzKUdSfUkXZHE418o6eEK9ukr6frtPO6anYvMbS9PYC4d1QNKTWBhJ1TnAE9gLj31A/aSNElSf0k9JH0i6QXgK0l54UCKAEi6XlLf8PVekt6TNF7Sp+E4ZWWSdJKkLyRNlPShpMYlNneQ9LGk6ZIuLvGeP0kaK2mKpFsq99Td9vDhdFw6ugHY38w6AkjqAXQJ182SlFfOex8HLjOz6ZIOBv4N9Cxn/1HAIWZmkn4H/Bm4LtzWHjgEqAlMlDQU2B9oHcYj4E1Jh5vZyB06U7dTPIG5uPjSzGaVt4OkWkBX4GVJxaurVnDcZsAgSblAFaBkGW+Y2TpgnaRPCJJWd6AXUDymfi2ChOYJLAKewFxcFJR4vYktmz+KR1vNAFYW19wS9BBwn5m9Gdb0+pbYtnUnSSOodd1pZv/ZjjJckngbmEtH+UDtcrYvAnaT1EBSVeBEADNbDcySdDqAAh0qKKsuUPxI0f9tte1kSdUkNQB6AGOB94GLwtoekppK2i3xU3OVyWtgLu2Y2TJJo8OG+neBoVtt3yjpVuALgku+kg9mnws8Gs46lA28BEwup7i+BJec84DPgZYltn0Zlr0HcJuZzQfmS9oXGBNepq4BzgMW7+Dpup3gjxI552LLLyGdc7HlCcw5F1uewJxzseUJzDkXW57AnHOx5QnMORdbnsCcc7HlCcw5F1v/D3uPyf4FZrLMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = confusion_matrix(y_test, predictions)\n",
    "sns.heatmap(cm, square=True, annot=True, cmap='RdBu', cbar=False,\n",
    "xticklabels=sections, yticklabels=sections)\n",
    "plt.xlabel('true label')\n",
    "plt.ylabel('predicted label')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}