NLP studies.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ddc3340d",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "547b67cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string\n",
    "import numpy as np \n",
    "import random\n",
    "import pandas as pd \n",
    "\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "from plotly import graph_objs as go\n",
    "#import plotly.express as px\n",
    "#import plotly.figure_factory as ff\n",
    "#from collections import Counter\n",
    "\n",
    "#from PIL import Image\n",
    "#from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n",
    "\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "#from tqdm import tqdm\n",
    "#import os\n",
    "#import spacy\n",
    "#from spacy.util import compounding\n",
    "#from spacy.util import minibatch\n",
    "\n",
    "#from collections import defaultdict\n",
    "#from collections import Counter\n",
    "\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.initializers import Constant\n",
    "from keras.layers import (LSTM, \n",
    "                          Embedding, \n",
    "                          BatchNormalization,\n",
    "                          Dense, \n",
    "                          TimeDistributed, \n",
    "                          Dropout, \n",
    "                          Bidirectional,\n",
    "                          Flatten, \n",
    "                          GlobalMaxPool1D)\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n",
    "#from keras.optimizer import Adam\n",
    "\n",
    "from sklearn.metrics import (\n",
    "    precision_score, \n",
    "    recall_score, \n",
    "    f1_score, \n",
    "    classification_report,\n",
    "    accuracy_score\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "18bbf093",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     C:\\Users\\isaac\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping corpora\\omw-1.4.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#nltk.download('stopwords')\n",
    "#nltk.download('punkt')\n",
    "#nltk.download('wordnet')\n",
    "#nltk.download('omw-1.4')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a551ee09",
   "metadata": {},
   "source": [
    "# 1. Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "0a4cb013",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  target                                            message\n",
       "0    ham  Go until jurong point, crazy.. Available only ...\n",
       "1    ham                      Ok lar... Joking wif u oni...\n",
       "2   spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3    ham  U dun say so early hor... U c already then say...\n",
       "4    ham  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"./data/spam.csv\", encoding=\"latin-1\")\n",
    "\n",
    "df = df.dropna(how=\"any\", axis=1)\n",
    "df.columns = ['target', 'message']\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "3bf9a3a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>message</th>\n",
       "      <th>message_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  target                                            message  message_len\n",
       "0    ham  Go until jurong point, crazy.. Available only ...           20\n",
       "1    ham                      Ok lar... Joking wif u oni...            6\n",
       "2   spam  Free entry in 2 a wkly comp to win FA Cup fina...           28\n",
       "3    ham  U dun say so early hor... U c already then say...           11\n",
       "4    ham  Nah I don't think he goes to usf, he lives aro...           13"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['message_len'] = df['message'].apply(lambda x: len(x.split(' ')))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "3d7781ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "171"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(df['message_len'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "090751aa",
   "metadata": {},
   "source": [
    "# 2. EDA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "5a9c24f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4825,  747], dtype=int64)"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# agg = Aggregate using one or more operations over the specified axis.\n",
    "balance_counts = df.groupby('target')['target'].agg('count').values\n",
    "balance_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "30fce72b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "name": "ham",
         "text": [
          "4825"
         ],
         "textposition": "auto",
         "type": "bar",
         "x": [
          "ham"
         ],
         "y": [
          4825
         ]
        },
        {
         "name": "spam",
         "text": [
          "747"
         ],
         "textposition": "auto",
         "type": "bar",
         "x": [
          "spam"
         ],
         "y": [
          747
         ]
        }
       ],
       "layout": {
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "<span style=\"font-size:32px; font-family:Times New Roman\">Dataset distribution by target</span>"
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"ceb02c6b-979d-4e24-8cb4-3244c39d9cf9\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"ceb02c6b-979d-4e24-8cb4-3244c39d9cf9\")) {                    Plotly.newPlot(                        \"ceb02c6b-979d-4e24-8cb4-3244c39d9cf9\",                        [{\"name\":\"ham\",\"text\":[\"4825\"],\"textposition\":\"auto\",\"x\":[\"ham\"],\"y\":[4825],\"type\":\"bar\"},{\"name\":\"spam\",\"text\":[\"747\"],\"textposition\":\"auto\",\"x\":[\"spam\"],\"y\":[747],\"type\":\"bar\"}],                        {\"template\":{\"data\":{\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"choropleth\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"type\":\"choropleth\"}],\"contour\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"contour\"}],\"contourcarpet\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"type\":\"contourcarpet\"}],\"heatmap\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"heatmap\"}],\"heatmapgl\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"heatmapgl\"}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"histogram2d\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"histogram2d\"}],\"histogram2dcontour\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"histogram2dcontour\"}],\"mesh3d\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"type\":\"mesh3d\"}],\"parcoords\":[{\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"parcoords\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}],\"scatter\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatter\"}],\"scatter3d\":[{\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatter3d\"}],\"scattercarpet\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattercarpet\"}],\"scattergeo\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattergeo\"}],\"scattergl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattergl\"}],\"scattermapbox\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattermapbox\"}],\"scatterpolar\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatterpolar\"}],\"scatterpolargl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatterpolargl\"}],\"scatterternary\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatterternary\"}],\"surface\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"surface\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}]},\"layout\":{\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"autotypenumbers\":\"strict\",\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]],\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]},\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"geo\":{\"bgcolor\":\"white\",\"lakecolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"showlakes\":true,\"showland\":true,\"subunitcolor\":\"white\"},\"hoverlabel\":{\"align\":\"left\"},\"hovermode\":\"closest\",\"mapbox\":{\"style\":\"light\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"bgcolor\":\"#E5ECF6\",\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"gridwidth\":2,\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\"},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"gridwidth\":2,\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\"},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"gridwidth\":2,\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\"}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"ternary\":{\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"bgcolor\":\"#E5ECF6\",\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"title\":{\"x\":0.05},\"xaxis\":{\"automargin\":true,\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"zerolinewidth\":2},\"yaxis\":{\"automargin\":true,\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"zerolinewidth\":2}}},\"title\":{\"text\":\"<span style=\\\"font-size:32px; font-family:Times New Roman\\\">Dataset distribution by target</span>\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('ceb02c6b-979d-4e24-8cb4-3244c39d9cf9');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = go.Figure()\n",
    "fig.add_trace(go.Bar(\n",
    "    x=['ham'],\n",
    "    y=[balance_counts[0]],\n",
    "    name='ham',\n",
    "    text=[balance_counts[0]],\n",
    "    textposition='auto',\n",
    "))\n",
    "fig.add_trace(go.Bar(\n",
    "    x=['spam'],\n",
    "    y=[balance_counts[1]],\n",
    "    name='spam',\n",
    "    text=[balance_counts[1]],\n",
    "    textposition='auto',\n",
    "))\n",
    "fig.update_layout(\n",
    "    title='<span style=\"font-size:32px; font-family:Times New Roman\">Dataset distribution by target</span>'\n",
    ")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9691b462",
   "metadata": {},
   "source": [
    "# 3. Data Pre-processing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "becd9278",
   "metadata": {},
   "source": [
    "- Now we are going to engineering the data to make it easier for the model to clasiffy.\n",
    "- This section is very important to reduce the dimensions of the problem."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08639ca3",
   "metadata": {},
   "source": [
    "### 3.1 Cleaning the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "bb9263de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Special thanks to https://www.kaggle.com/tanulsingh077 for this function\n",
    "def clean_text(text):\n",
    "    '''Make text lowercase, remove text in square brackets,remove links,remove punctuation\n",
    "    and remove words containing numbers.'''\n",
    "    text = str(text).lower()\n",
    "    text = re.sub('\\[.*?\\]', '', text)\n",
    "    text = re.sub('https?://\\S+|www\\.\\S+', '', text)\n",
    "    text = re.sub('<.*?>+', '', text)\n",
    "    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)\n",
    "    text = re.sub('\\n', '', text)\n",
    "    text = re.sub('\\w*\\d\\w*', '', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5d5b0e43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>message</th>\n",
       "      <th>message_len</th>\n",
       "      <th>message_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>20</td>\n",
       "      <td>go until jurong point crazy available only in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>6</td>\n",
       "      <td>ok lar joking wif u oni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>28</td>\n",
       "      <td>free entry in  a wkly comp to win fa cup final...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>11</td>\n",
       "      <td>u dun say so early hor u c already then say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>13</td>\n",
       "      <td>nah i dont think he goes to usf he lives aroun...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  target                                            message  message_len  \\\n",
       "0    ham  Go until jurong point, crazy.. Available only ...           20   \n",
       "1    ham                      Ok lar... Joking wif u oni...            6   \n",
       "2   spam  Free entry in 2 a wkly comp to win FA Cup fina...           28   \n",
       "3    ham  U dun say so early hor... U c already then say...           11   \n",
       "4    ham  Nah I don't think he goes to usf, he lives aro...           13   \n",
       "\n",
       "                                       message_clean  \n",
       "0  go until jurong point crazy available only in ...  \n",
       "1                            ok lar joking wif u oni  \n",
       "2  free entry in  a wkly comp to win fa cup final...  \n",
       "3        u dun say so early hor u c already then say  \n",
       "4  nah i dont think he goes to usf he lives aroun...  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['message_clean'] = df['message'].apply(clean_text)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "a8fd7c73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>message</th>\n",
       "      <th>message_len</th>\n",
       "      <th>message_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>20</td>\n",
       "      <td>go jurong point crazy available bugis n great ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>6</td>\n",
       "      <td>ok lar joking wif oni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>28</td>\n",
       "      <td>free entry  wkly comp win fa cup final tkts  m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>11</td>\n",
       "      <td>dun say early hor already say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>13</td>\n",
       "      <td>nah dont think goes usf lives around though</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  target                                            message  message_len  \\\n",
       "0    ham  Go until jurong point, crazy.. Available only ...           20   \n",
       "1    ham                      Ok lar... Joking wif u oni...            6   \n",
       "2   spam  Free entry in 2 a wkly comp to win FA Cup fina...           28   \n",
       "3    ham  U dun say so early hor... U c already then say...           11   \n",
       "4    ham  Nah I don't think he goes to usf, he lives aro...           13   \n",
       "\n",
       "                                       message_clean  \n",
       "0  go jurong point crazy available bugis n great ...  \n",
       "1                              ok lar joking wif oni  \n",
       "2  free entry  wkly comp win fa cup final tkts  m...  \n",
       "3                      dun say early hor already say  \n",
       "4        nah dont think goes usf lives around though  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words = stopwords.words('english')\n",
    "more_stopwords = ['u', 'im', 'c']\n",
    "stop_words = stop_words + more_stopwords\n",
    "\n",
    "def remove_stopwords(text):\n",
    "    text = ' '.join(word for word in text.split(' ') if word not in stop_words)\n",
    "    return text\n",
    "    \n",
    "df['message_clean'] = df['message_clean'].apply(remove_stopwords)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "046b0bc1",
   "metadata": {},
   "source": [
    "### 3.2 Stemming "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8c79d6f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = nltk.SnowballStemmer(\"english\")\n",
    "\n",
    "def stemm_text(text):\n",
    "    text = ' '.join(stemmer.stem(word) for word in text.split(' '))\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "25753cfe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>message</th>\n",
       "      <th>message_len</th>\n",
       "      <th>message_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>20</td>\n",
       "      <td>go jurong point crazi avail bugi n great world...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>6</td>\n",
       "      <td>ok lar joke wif oni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>28</td>\n",
       "      <td>free entri  wkli comp win fa cup final tkts  m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>11</td>\n",
       "      <td>dun say earli hor alreadi say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>13</td>\n",
       "      <td>nah dont think goe usf live around though</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  target                                            message  message_len  \\\n",
       "0    ham  Go until jurong point, crazy.. Available only ...           20   \n",
       "1    ham                      Ok lar... Joking wif u oni...            6   \n",
       "2   spam  Free entry in 2 a wkly comp to win FA Cup fina...           28   \n",
       "3    ham  U dun say so early hor... U c already then say...           11   \n",
       "4    ham  Nah I don't think he goes to usf, he lives aro...           13   \n",
       "\n",
       "                                       message_clean  \n",
       "0  go jurong point crazi avail bugi n great world...  \n",
       "1                                ok lar joke wif oni  \n",
       "2  free entri  wkli comp win fa cup final tkts  m...  \n",
       "3                      dun say earli hor alreadi say  \n",
       "4          nah dont think goe usf live around though  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['message_clean'] = df['message_clean'].apply(stemm_text)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c27c256f",
   "metadata": {},
   "source": [
    "### 3.3 All together "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c5dfe086",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(text):\n",
    "    # Clean puntuation, urls, and so on\n",
    "    text = clean_text(text)\n",
    "    # Remove stopwords\n",
    "    text = ' '.join(word for word in text.split(' ') if word not in stop_words)\n",
    "    # Stemm all the words in the sentence\n",
    "    text = ' '.join(stemmer.stem(word) for word in text.split(' '))\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "c8bfeacd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>message</th>\n",
       "      <th>message_len</th>\n",
       "      <th>message_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>20</td>\n",
       "      <td>go jurong point crazi avail bugi n great world...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>6</td>\n",
       "      <td>ok lar joke wif oni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>28</td>\n",
       "      <td>free entri  wkli comp win fa cup final tkts  m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>11</td>\n",
       "      <td>dun say ear hor alreadi say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>13</td>\n",
       "      <td>nah dont think goe usf live around though</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  target                                            message  message_len  \\\n",
       "0    ham  Go until jurong point, crazy.. Available only ...           20   \n",
       "1    ham                      Ok lar... Joking wif u oni...            6   \n",
       "2   spam  Free entry in 2 a wkly comp to win FA Cup fina...           28   \n",
       "3    ham  U dun say so early hor... U c already then say...           11   \n",
       "4    ham  Nah I don't think he goes to usf, he lives aro...           13   \n",
       "\n",
       "                                       message_clean  \n",
       "0  go jurong point crazi avail bugi n great world...  \n",
       "1                                ok lar joke wif oni  \n",
       "2  free entri  wkli comp win fa cup final tkts  m...  \n",
       "3                        dun say ear hor alreadi say  \n",
       "4          nah dont think goe usf live around though  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['message_clean'] = df['message_clean'].apply(preprocess_data)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fabfca2",
   "metadata": {},
   "source": [
    "### 3.4 Target encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "556c1320",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>message</th>\n",
       "      <th>message_len</th>\n",
       "      <th>message_clean</th>\n",
       "      <th>target_encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>20</td>\n",
       "      <td>go jurong point crazi avail bugi n great world...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>6</td>\n",
       "      <td>ok lar joke wif oni</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>28</td>\n",
       "      <td>free entri  wkli comp win fa cup final tkts  m...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>11</td>\n",
       "      <td>dun say ear hor alreadi say</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>13</td>\n",
       "      <td>nah dont think goe usf live around though</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  target                                            message  message_len  \\\n",
       "0    ham  Go until jurong point, crazy.. Available only ...           20   \n",
       "1    ham                      Ok lar... Joking wif u oni...            6   \n",
       "2   spam  Free entry in 2 a wkly comp to win FA Cup fina...           28   \n",
       "3    ham  U dun say so early hor... U c already then say...           11   \n",
       "4    ham  Nah I don't think he goes to usf, he lives aro...           13   \n",
       "\n",
       "                                       message_clean  target_encoded  \n",
       "0  go jurong point crazi avail bugi n great world...               0  \n",
       "1                                ok lar joke wif oni               0  \n",
       "2  free entri  wkli comp win fa cup final tkts  m...               1  \n",
       "3                        dun say ear hor alreadi say               0  \n",
       "4          nah dont think goe usf live around though               0  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "le = LabelEncoder()\n",
    "le.fit(df['target'])\n",
    "\n",
    "df['target_encoded'] = le.transform(df['target'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0e75dd5",
   "metadata": {},
   "source": [
    "# 4. Tokens visualization "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd6bb258",
   "metadata": {},
   "source": [
    "# 5. Vectorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "2ccb2c67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5572 , 5572\n"
     ]
    }
   ],
   "source": [
    "# how to define X and y (from the SMS data) for use with CountVectorizer\n",
    "x = df['message_clean']\n",
    "y = df['target_encoded']\n",
    "\n",
    "\n",
    "print(len(x), \",\" ,len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "e487f336",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4179 4179\n",
      "1393 1393\n"
     ]
    }
   ],
   "source": [
    "# Split into train and test sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=42)\n",
    "\n",
    "\n",
    "print(len(x_train), len(y_train))\n",
    "print(len(x_test), len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "2bcbd3f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer()"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "# CountVectorizer conta quantos tokens, de cada tipo, aparecem em cada mensagem e criam um vetor para cada mensagem.\n",
    "vect = CountVectorizer()\n",
    "vect.fit(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "32e1d46c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4179, 5684)\n",
      "(1393, 5684)\n"
     ]
    }
   ],
   "source": [
    "# Use the trained to create a document-term matrix from train and test sets\n",
    "x_train_dtm = vect.transform(x_train)\n",
    "x_test_dtm = vect.transform(x_test)\n",
    "\n",
    "print(x_train_dtm.shape)\n",
    "print(x_test_dtm.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e881186",
   "metadata": {},
   "source": [
    "### 5.1 Tunning CountVectorizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b5bcf74",
   "metadata": {},
   "source": [
    "1. stop_words: Stop words are just a list of words you don’t want to use as features. (stop_words=’english’ to use a built-in list. Alternatively you can set stop_words equal to some custom list)\n",
    "\n",
    "\n",
    "2. ngram_range: Set the parameter ngram_range=(a,b) where a is the minimum and b is the maximum size of ngrams you want to include in your features.\n",
    "\n",
    "\n",
    "3. min_df, max_df: These are the minimum and maximum document frequencies words/n-grams must have to be used as features. (If either is set to a float, that number will be interpreted as a frequency rather than a numerical limit. min_df defaults to 1 (int) and max_df defaults to 1.0 (float))\n",
    "\n",
    "\n",
    "4. max_features: This parameter is pretty self-explanatory. The CountVectorizer will choose the words/features that occur most frequently to be in its’ vocabulary and drop everything else."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "d02c0a30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(max_features=300, min_df=10, ngram_range=(1, 2),\n",
       "                stop_words='english')"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vect_tunned = CountVectorizer(stop_words='english', ngram_range=(1,2), min_df=10, max_df=1.0, max_features=300)\n",
    "vect_tunned.fit(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "30719539",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4179, 300)\n",
      "(1393, 300)\n"
     ]
    }
   ],
   "source": [
    "# Use the trained to create a document-term matrix from train and test sets\n",
    "x_train_dtm_tunned = vect_tunned.transform(x_train)\n",
    "x_test_dtm_tunned = vect_tunned.transform(x_test)\n",
    "\n",
    "print(x_train_dtm_tunned.shape)\n",
    "print(x_test_dtm_tunned.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a96e58e",
   "metadata": {},
   "source": [
    "### 5.2 TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "b058540f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfTransformer()"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Matriz documento-termo que considera a frequencia que os termos aparecem nos documentos.\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "TF = TfidfTransformer()\n",
    "TF.fit(x_train_dtm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "7832155d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4179, 5684)\n",
      "(1393, 5684)\n"
     ]
    }
   ],
   "source": [
    "x_train_tfidf = TF.transform(x_train_dtm)\n",
    "x_test_tfidf = TF.transform(x_test_dtm)\n",
    "\n",
    "print(x_train_tfidf.shape)\n",
    "print(x_test_tfidf.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e0b90ae",
   "metadata": {},
   "source": [
    "### 5.3 Word Embeddings: GloVe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "d4a51ed5",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = df['message_clean']\n",
    "target = df['target_encoded']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "0a2f6739",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6726"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tokenizer is from Keras.preprocessing.text\n",
    "# Calculate the length of our vocabulary\n",
    "\n",
    "word_tokenizer = Tokenizer()\n",
    "word_tokenizer.fit_on_texts(texts)\n",
    "\n",
    "vocab_length = len(word_tokenizer.word_index) + 1\n",
    "vocab_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "8460b591",
   "metadata": {},
   "outputs": [],
   "source": [
    "def embed(text): \n",
    "    return word_tokenizer.texts_to_sequences(text)\n",
    "\n",
    "longest_train = max(texts, key=lambda sentence: len(word_tokenize(sentence)))\n",
    "#longest_train\n",
    "length_long_sentence = len(word_tokenize(longest_train))\n",
    "#length_long_sentence"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81ffd135",
   "metadata": {},
   "source": [
    "##### 5.3.1 Pad_sequences\n",
    "https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "445748d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5572, 80)"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fazendo uma matriz onde cada linha é a tokenização de um texto.\n",
    "list_of_lists = embed(texts)\n",
    "\n",
    "train_padded_sentences = pad_sequences(\n",
    "    list_of_lists, \n",
    "    length_long_sentence, \n",
    "    padding='post'\n",
    ")\n",
    "\n",
    "train_padded_sentences.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa7823d2",
   "metadata": {},
   "source": [
    "##### 5.3.2 GloVe\n",
    "\n",
    "- GloVe method is built on an important idea:\n",
    "    - You can derive semantic relationships between words from the co-occurrence matrix.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "f80a91c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_dictionary = dict()\n",
    "embedding_dim = 100\n",
    "\n",
    "# Load GloVe 100D embeddings\n",
    "with open('./data/glove.6B.100d.txt', encoding=\"utf8\") as fp:\n",
    "    for line in fp.readlines():\n",
    "        records = line.split()\n",
    "        word = records[0]\n",
    "        vector_dimensions = np.asarray(records[1:], dtype='float32')\n",
    "        embeddings_dictionary [word] = vector_dimensions\n",
    "\n",
    "#embeddings_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "97f548df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6726, 100)"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we will load embedding vectors of those words that appear in the\n",
    "# Glove dictionary. Others will be initialized to 0.\n",
    "\n",
    "embedding_matrix = np.zeros((vocab_length, embedding_dim))\n",
    "\n",
    "for word, index in word_tokenizer.word_index.items():\n",
    "    embedding_vector = embeddings_dictionary.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[index] = embedding_vector\n",
    "        \n",
    "embedding_matrix.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b3eb074",
   "metadata": {},
   "source": [
    "# 6. Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "36bec619",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB()"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a Multinomial Naive Bayes model\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "nb = MultinomialNB()\n",
    "\n",
    "# Treinando o modelo com os textos vetorizados pelo CountVectorizer\n",
    "nb.fit(x_train_tfidf, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "984b23e4",
   "metadata": {},
   "source": [
    "### 6.1 Naive Bayes DTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "88586b98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1393,)"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make class and probability predictions\n",
    "y_pred_class = nb.predict(x_test_dtm)\n",
    "y_pred_prob = nb.predict_proba(x_test_dtm)[:, 1]\n",
    "\n",
    "y_pred_class.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "32431dcd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9748743718592965\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1199,    3],\n",
       "       [  32,  159]], dtype=int64)"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate accuracy of class predictions\n",
    "from sklearn import metrics\n",
    "print(metrics.accuracy_score(y_test, y_pred_class))\n",
    "\n",
    "metrics.confusion_matrix(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06352739",
   "metadata": {},
   "outputs": [],
   "source": [
    "0.9784637473079684\n",
    "array([[1191,   11],\n",
    "       [  19,  172]], dtype=int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "e31bcdd3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.974296765425861"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate AUC\n",
    "metrics.roc_auc_score(y_test, y_pred_prob)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c599635d",
   "metadata": {},
   "source": [
    "### 6.2 Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "8890273d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "pipe = Pipeline([('bow', CountVectorizer()), \n",
    "                 ('tfid', TfidfTransformer()),  \n",
    "                 ('model', MultinomialNB())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "38282450",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9597989949748744\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1202,    0],\n",
       "       [  56,  135]], dtype=int64)"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the pipeline with the data\n",
    "pipe.fit(x_train, y_train)\n",
    "\n",
    "y_pred_class = pipe.predict(x_test)\n",
    "\n",
    "print(metrics.accuracy_score(y_test, y_pred_class))\n",
    "\n",
    "metrics.confusion_matrix(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1289a373",
   "metadata": {},
   "source": [
    "### 6.3 XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "8531a49e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('bow', CountVectorizer()), \n",
    "    ('tfid', TfidfTransformer()),  \n",
    "    ('model', xgb.XGBClassifier(\n",
    "        learning_rate=0.1,\n",
    "        max_depth=7,\n",
    "        n_estimators=80,\n",
    "        use_label_encoder=False,\n",
    "        eval_metric='auc',\n",
    "        # colsample_bytree=0.8,\n",
    "        # subsample=0.7,\n",
    "        # min_child_weight=5,\n",
    "    ))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "8425803c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: 0.9834888729361091\n",
      "Test: 0.9662598707824839\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1198,    4],\n",
       "       [  43,  148]], dtype=int64)"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the pipeline with the data\n",
    "pipe.fit(x_train, y_train)\n",
    "\n",
    "y_pred_class = pipe.predict(x_test)\n",
    "y_pred_train = pipe.predict(x_train)\n",
    "\n",
    "print('Train: {}'.format(metrics.accuracy_score(y_train, y_pred_train)))\n",
    "print('Test: {}'.format(metrics.accuracy_score(y_test, y_pred_class)))\n",
    "\n",
    "metrics.confusion_matrix(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adcf1de3",
   "metadata": {},
   "source": [
    "# 7. LSTM (Deep learning model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "a40a7f4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split data into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    train_padded_sentences, \n",
    "    target, \n",
    "    test_size=0.25\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "9308addc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding (Embedding)       (None, 80, 100)           672600    \n",
      "                                                                 \n",
      " bidirectional (Bidirectiona  (None, 80, 160)          115840    \n",
      " l)                                                              \n",
      "                                                                 \n",
      " global_max_pooling1d (Globa  (None, 160)              0         \n",
      " lMaxPooling1D)                                                  \n",
      "                                                                 \n",
      " batch_normalization (BatchN  (None, 160)              640       \n",
      " ormalization)                                                   \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 160)               0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 80)                12880     \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 80)                0         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 80)                6480      \n",
      "                                                                 \n",
      " dropout_2 (Dropout)         (None, 80)                0         \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 1)                 81        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 808,521\n",
      "Trainable params: 808,201\n",
      "Non-trainable params: 320\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Model from https://www.kaggle.com/mariapushkareva/nlp-disaster-tweets-with-glove-and-lstm/data\n",
    "\n",
    "def glove_lstm():\n",
    "    model = Sequential()\n",
    "    \n",
    "    model.add(Embedding(\n",
    "        input_dim=embedding_matrix.shape[0], \n",
    "        output_dim=embedding_matrix.shape[1], \n",
    "        weights = [embedding_matrix], \n",
    "        input_length=length_long_sentence\n",
    "    ))\n",
    "    \n",
    "    model.add(Bidirectional(LSTM(\n",
    "        length_long_sentence, \n",
    "        return_sequences = True, \n",
    "        recurrent_dropout=0.2\n",
    "    )))\n",
    "    \n",
    "    model.add(GlobalMaxPool1D())\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(length_long_sentence, activation = \"relu\"))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(length_long_sentence, activation = \"relu\"))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(1, activation = 'sigmoid'))\n",
    "    model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "model = glove_lstm()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "79e70a45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/7\n",
      "131/131 [==============================] - ETA: 0s - loss: 0.3024 - accuracy: 0.8818\n",
      "Epoch 00001: val_loss improved from inf to 0.40072, saving model to model.h5\n",
      "131/131 [==============================] - 25s 139ms/step - loss: 0.3024 - accuracy: 0.8818 - val_loss: 0.4007 - val_accuracy: 0.9605 - lr: 0.0010\n",
      "Epoch 2/7\n",
      "131/131 [==============================] - ETA: 0s - loss: 0.1754 - accuracy: 0.9414\n",
      "Epoch 00002: val_loss improved from 0.40072 to 0.16010, saving model to model.h5\n",
      "131/131 [==============================] - 17s 130ms/step - loss: 0.1754 - accuracy: 0.9414 - val_loss: 0.1601 - val_accuracy: 0.9706 - lr: 0.0010\n",
      "Epoch 3/7\n",
      "131/131 [==============================] - ETA: 0s - loss: 0.1161 - accuracy: 0.9624\n",
      "Epoch 00003: val_loss improved from 0.16010 to 0.10229, saving model to model.h5\n",
      "131/131 [==============================] - 17s 129ms/step - loss: 0.1161 - accuracy: 0.9624 - val_loss: 0.1023 - val_accuracy: 0.9763 - lr: 0.0010\n",
      "Epoch 4/7\n",
      "131/131 [==============================] - ETA: 0s - loss: 0.0960 - accuracy: 0.9710\n",
      "Epoch 00004: val_loss improved from 0.10229 to 0.09631, saving model to model.h5\n",
      "131/131 [==============================] - 17s 129ms/step - loss: 0.0960 - accuracy: 0.9710 - val_loss: 0.0963 - val_accuracy: 0.9806 - lr: 0.0010\n",
      "Epoch 5/7\n",
      "131/131 [==============================] - ETA: 0s - loss: 0.0752 - accuracy: 0.9794\n",
      "Epoch 00005: val_loss did not improve from 0.09631\n",
      "131/131 [==============================] - 19s 142ms/step - loss: 0.0752 - accuracy: 0.9794 - val_loss: 0.1068 - val_accuracy: 0.9770 - lr: 0.0010\n",
      "Epoch 6/7\n",
      "131/131 [==============================] - ETA: 0s - loss: 0.0768 - accuracy: 0.9785\n",
      "Epoch 00006: val_loss did not improve from 0.09631\n",
      "131/131 [==============================] - 18s 140ms/step - loss: 0.0768 - accuracy: 0.9785 - val_loss: 0.1189 - val_accuracy: 0.9641 - lr: 0.0010\n",
      "Epoch 7/7\n",
      "131/131 [==============================] - ETA: 0s - loss: 0.0715 - accuracy: 0.9789\n",
      "Epoch 00007: val_loss did not improve from 0.09631\n",
      "131/131 [==============================] - 20s 150ms/step - loss: 0.0715 - accuracy: 0.9789 - val_loss: 0.1051 - val_accuracy: 0.9799 - lr: 0.0010\n"
     ]
    }
   ],
   "source": [
    "# Load the model and train!!\n",
    "\n",
    "model = glove_lstm()\n",
    "\n",
    "checkpoint = ModelCheckpoint(\n",
    "    'model.h5', \n",
    "    monitor = 'val_loss', \n",
    "    verbose = 1, \n",
    "    save_best_only = True\n",
    ")\n",
    "reduce_lr = ReduceLROnPlateau(\n",
    "    monitor = 'val_loss', \n",
    "    factor = 0.2, \n",
    "    verbose = 1, \n",
    "    patience = 5,                        \n",
    "    min_lr = 0.001\n",
    ")\n",
    "history = model.fit(\n",
    "    X_train, \n",
    "    y_train, \n",
    "    epochs = 7,\n",
    "    batch_size = 32,\n",
    "    validation_data = (X_test, y_test),\n",
    "    verbose = 1,\n",
    "    callbacks = [reduce_lr, checkpoint]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "76dffebe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9798994974874372\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1195,   10],\n",
       "       [  18,  170]], dtype=int64)"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_preds = (model.predict(X_test) > 0.5).astype(\"int32\")\n",
    "\n",
    "print(metrics.accuracy_score(y_test, y_preds))\n",
    "metrics.confusion_matrix(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "330605aa",
   "metadata": {},
   "source": [
    "- Melhor modelo até o momento"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecb46e74",
   "metadata": {},
   "source": [
    "# 8. NLP: Disaster Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "946ac657",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "      <th>text_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                               text  target  text_len\n",
       "0   1  Our Deeds are the Reason of this #earthquake M...       1        13\n",
       "1   4             Forest fire near La Ronge Sask. Canada       1         7\n",
       "2   5  All residents asked to 'shelter in place' are ...       1        22\n",
       "3   6  13,000 people receive #wildfires evacuation or...       1         9\n",
       "4   7  Just got sent this photo from Ruby #Alaska as ...       1        17"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"./data/train.csv\", encoding=\"latin-1\")\n",
    "test_df = pd.read_csv(\"./data/test.csv\", encoding=\"latin-1\")\n",
    "\n",
    "df = df.dropna(how=\"any\", axis=1)\n",
    "df['text_len'] = df['text'].apply(lambda x: len(x.split(' ')))\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60d4f25b",
   "metadata": {},
   "source": [
    "### 8.1 EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "c18e9a1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4342, 3271], dtype=int64)"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "balance_counts = df.groupby('target')['target'].agg('count').values\n",
    "balance_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "6f2d31f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "name": "Fake",
         "text": [
          "4342"
         ],
         "textposition": "auto",
         "type": "bar",
         "x": [
          "Fake"
         ],
         "y": [
          4342
         ]
        },
        {
         "name": "Real disaster",
         "text": [
          "3271"
         ],
         "textposition": "auto",
         "type": "bar",
         "x": [
          "Real disaster"
         ],
         "y": [
          3271
         ]
        }
       ],
       "layout": {
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "<span style=\"font-size:32px; font-family:Times New Roman\">Dataset distribution by target</span>"
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"603b1774-a404-481d-a2de-1863fcf79fe7\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"603b1774-a404-481d-a2de-1863fcf79fe7\")) {                    Plotly.newPlot(                        \"603b1774-a404-481d-a2de-1863fcf79fe7\",                        [{\"name\":\"Fake\",\"text\":[\"4342\"],\"textposition\":\"auto\",\"x\":[\"Fake\"],\"y\":[4342],\"type\":\"bar\"},{\"name\":\"Real disaster\",\"text\":[\"3271\"],\"textposition\":\"auto\",\"x\":[\"Real disaster\"],\"y\":[3271],\"type\":\"bar\"}],                        {\"template\":{\"data\":{\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"choropleth\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"type\":\"choropleth\"}],\"contour\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"contour\"}],\"contourcarpet\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"type\":\"contourcarpet\"}],\"heatmap\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"heatmap\"}],\"heatmapgl\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"heatmapgl\"}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"histogram2d\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"histogram2d\"}],\"histogram2dcontour\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"histogram2dcontour\"}],\"mesh3d\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"type\":\"mesh3d\"}],\"parcoords\":[{\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"parcoords\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}],\"scatter\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatter\"}],\"scatter3d\":[{\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatter3d\"}],\"scattercarpet\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattercarpet\"}],\"scattergeo\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattergeo\"}],\"scattergl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattergl\"}],\"scattermapbox\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattermapbox\"}],\"scatterpolar\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatterpolar\"}],\"scatterpolargl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatterpolargl\"}],\"scatterternary\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatterternary\"}],\"surface\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"surface\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}]},\"layout\":{\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"autotypenumbers\":\"strict\",\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]],\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]},\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"geo\":{\"bgcolor\":\"white\",\"lakecolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"showlakes\":true,\"showland\":true,\"subunitcolor\":\"white\"},\"hoverlabel\":{\"align\":\"left\"},\"hovermode\":\"closest\",\"mapbox\":{\"style\":\"light\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"bgcolor\":\"#E5ECF6\",\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"gridwidth\":2,\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\"},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"gridwidth\":2,\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\"},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"gridwidth\":2,\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\"}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"ternary\":{\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"bgcolor\":\"#E5ECF6\",\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"title\":{\"x\":0.05},\"xaxis\":{\"automargin\":true,\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"zerolinewidth\":2},\"yaxis\":{\"automargin\":true,\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"zerolinewidth\":2}}},\"title\":{\"text\":\"<span style=\\\"font-size:32px; font-family:Times New Roman\\\">Dataset distribution by target</span>\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('603b1774-a404-481d-a2de-1863fcf79fe7');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = go.Figure()\n",
    "fig.add_trace(go.Bar(\n",
    "    x=['Fake'],\n",
    "    y=[balance_counts[0]],\n",
    "    name='Fake',\n",
    "    text=[balance_counts[0]],\n",
    "    textposition='auto',\n",
    "))\n",
    "fig.add_trace(go.Bar(\n",
    "    x=['Real disaster'],\n",
    "    y=[balance_counts[1]],\n",
    "    name='Real disaster',\n",
    "    text=[balance_counts[1]],\n",
    "    textposition='auto',\n",
    "))\n",
    "fig.update_layout(\n",
    "    title='<span style=\"font-size:32px; font-family:Times New Roman\">Dataset distribution by target</span>'\n",
    ")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b01246a",
   "metadata": {},
   "source": [
    "### 8.2 Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "fad486af",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_url(text):\n",
    "    url = re.compile(r'https?://\\S+|www\\.\\S+')\n",
    "    return url.sub(r'', text)\n",
    "\n",
    "\n",
    "def remove_emoji(text):\n",
    "    emoji_pattern = re.compile(\n",
    "        '['\n",
    "        u'\\U0001F600-\\U0001F64F'  # emoticons\n",
    "        u'\\U0001F300-\\U0001F5FF'  # symbols & pictographs\n",
    "        u'\\U0001F680-\\U0001F6FF'  # transport & map symbols\n",
    "        u'\\U0001F1E0-\\U0001F1FF'  # flags (iOS)\n",
    "        u'\\U00002702-\\U000027B0'\n",
    "        u'\\U000024C2-\\U0001F251'\n",
    "        ']+',\n",
    "        flags=re.UNICODE)\n",
    "    return emoji_pattern.sub(r'', text)\n",
    "\n",
    "\n",
    "def remove_html(text):\n",
    "    html = re.compile(r'<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});')\n",
    "    return re.sub(html, '', text)\n",
    "\n",
    "# Special thanks to https://www.kaggle.com/tanulsingh077 for this function\n",
    "def clean_text(text):\n",
    "    \n",
    "    '''Make text lowercase, remove text in square brackets,remove links,remove punctuation\n",
    "    and remove words containing numbers.'''\n",
    "    \n",
    "    text = str(text).lower()\n",
    "    text = re.sub('\\[.*?\\]', '', text)\n",
    "    text = re.sub(\n",
    "        'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', \n",
    "        '', \n",
    "        text\n",
    "    )\n",
    "    text = re.sub('https?://\\S+|www\\.\\S+', '', text)\n",
    "    text = re.sub('<.*?>+', '', text)\n",
    "    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)\n",
    "    text = re.sub('\\n', '', text)\n",
    "    text = re.sub('\\w*\\d\\w*', '', text)\n",
    "    \n",
    "    text = remove_url(text)\n",
    "    text = remove_emoji(text)\n",
    "    text = remove_html(text)\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "2141d8fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Omg another Earthquake '"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Test emoji removal\n",
    "remove_emoji(\"Omg another Earthquake 😔😔\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "id": "5db2939f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stops words\n",
    "\n",
    "stop_words = stopwords.words('english')\n",
    "more_stopwords = ['u', 'im', 'c']\n",
    "stop_words = stop_words + more_stopwords\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Stemming\n",
    "\n",
    "stemmer = nltk.SnowballStemmer(\"english\")\n",
    "\n",
    "\n",
    "def preprocess_data(text):\n",
    "    # Clean puntuation, urls, and so on\n",
    "    text = clean_text(text)\n",
    "    # Remove stopwords and Stemm all the words in the sentence\n",
    "    text = ' '.join(stemmer.stem(word) for word in text.split(' ') if word not in stop_words)\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "id": "4d227309",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "      <th>text_len</th>\n",
       "      <th>text_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>deeds reason earthquake may allah forgive us</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>forest fire near la ronge sask canada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>residents asked shelter place notified officer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>people receive wildfires evacuation orders ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>got sent photo ruby alaska smoke wildfires pou...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                               text  target  text_len  \\\n",
       "0   1  Our Deeds are the Reason of this #earthquake M...       1        13   \n",
       "1   4             Forest fire near La Ronge Sask. Canada       1         7   \n",
       "2   5  All residents asked to 'shelter in place' are ...       1        22   \n",
       "3   6  13,000 people receive #wildfires evacuation or...       1         9   \n",
       "4   7  Just got sent this photo from Ruby #Alaska as ...       1        17   \n",
       "\n",
       "                                          text_clean  \n",
       "0       deeds reason earthquake may allah forgive us  \n",
       "1              forest fire near la ronge sask canada  \n",
       "2  residents asked shelter place notified officer...  \n",
       "3   people receive wildfires evacuation orders ca...  \n",
       "4  got sent photo ruby alaska smoke wildfires pou...  "
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df['text_clean'] = test_df['text'].apply(preprocess_data)\n",
    "\n",
    "df['text_clean'] = df['text'].apply(preprocess_data)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "045a7774",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "badd5ed4",
   "metadata": {},
   "source": [
    "### 8.3 WordCloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "e71b2dff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_corpus_df(tweet, target):\n",
    "    corpus=[]\n",
    "    \n",
    "    # Sem o \".str\", o x é uma serie e não possui o metodos \".split\"\n",
    "    for x in tweet[tweet['target']==target]['text_clean'].str.split():\n",
    "        for i in x:\n",
    "            corpus.append(i)\n",
    "            break\n",
    "    return corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b74a422e",
   "metadata": {},
   "source": [
    "- Real disasters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "id": "565cfe9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('latest', 33),\n",
       " ('watch', 30),\n",
       " ('wreckage', 27),\n",
       " ('police', 25),\n",
       " ('families', 25),\n",
       " ('suicide', 24),\n",
       " ('rt', 23),\n",
       " ('years', 23),\n",
       " ('news', 22),\n",
       " ('california', 22)]"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_disaster_tweets = create_corpus_df(df, 1)\n",
    "\n",
    "dic={}\n",
    "for word in corpus_disaster_tweets:\n",
    "    if word in dic:\n",
    "        dic[word]+=1\n",
    "    else:\n",
    "        dic[word]=1\n",
    "        \n",
    "top=sorted(dic.items(), key=lambda x:x[1],reverse=True)[:10]\n",
    "top"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9bf3cfd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "47bacba2",
   "metadata": {},
   "source": [
    "- Fake disasters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "id": "b7eb4bfe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('new', 58),\n",
       " ('dont', 35),\n",
       " ('liked', 34),\n",
       " ('rt', 25),\n",
       " ('one', 20),\n",
       " ('love', 19),\n",
       " ('people', 17),\n",
       " ('hot', 17),\n",
       " ('reddit', 17),\n",
       " ('feel', 16)]"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_fake_disaster_tweets = create_corpus_df(df, 0)\n",
    "\n",
    "dic={}\n",
    "for word in corpus_fake_disaster_tweets:\n",
    "    if word in dic:\n",
    "        dic[word]+=1\n",
    "    else:\n",
    "        dic[word]=1\n",
    "        \n",
    "top=sorted(dic.items(), key=lambda x:x[1],reverse=True)[:10]\n",
    "top"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be195eb9",
   "metadata": {},
   "source": [
    "### 8.4 Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "id": "0cad81f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5709 5709\n",
      "1904 1904\n"
     ]
    }
   ],
   "source": [
    "# how to define X and y (from the SMS data) for use with COUNTVECTORIZER\n",
    "x = df['text_clean']\n",
    "y = df['target']\n",
    "\n",
    "# Split into train and test sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=42)\n",
    "print(len(x_train), len(y_train))\n",
    "print(len(x_test), len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "id": "07f1d6dd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pipe = Pipeline([\n",
    "    ('count', CountVectorizer()), \n",
    "    ('tfid', TfidfTransformer()),  \n",
    "    ('model', xgb.XGBClassifier(\n",
    "        use_label_encoder=False,\n",
    "        eval_metric='auc',\n",
    "    ))\n",
    "])\n",
    "from sklearn import metrics\n",
    "\n",
    "# Fit the pipeline with the data\n",
    "pipe = pipe.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "7901d7ef",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test: 0.7715336134453782\n",
      "[[970 121]\n",
      " [314 499]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        fake       0.76      0.89      0.82      1091\n",
      "    disaster       0.80      0.61      0.70       813\n",
      "\n",
      "    accuracy                           0.77      1904\n",
      "   macro avg       0.78      0.75      0.76      1904\n",
      "weighted avg       0.78      0.77      0.77      1904\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "y_pred_class = pipe.predict(x_test)\n",
    "\n",
    "print('Test: {}'.format(metrics.accuracy_score(y_test, y_pred_class)))\n",
    "\n",
    "print(metrics.confusion_matrix(y_test, y_pred_class))\n",
    "\n",
    "print(classification_report(y_test, y_pred_class, target_names=['fake', 'disaster']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c646515",
   "metadata": {},
   "source": [
    "- Resultado melhor que o anterior"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09cb50a7",
   "metadata": {},
   "source": [
    "### Saving and submiting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "id": "44bdb6e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = pd.read_csv(\"./data/sample_submission.csv\", index_col=\"id\")\n",
    "y_submit = pipe.predict(test_df['text_clean'])\n",
    "sample['target'] = y_submit\n",
    "sample.to_csv(\"third_submission.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64010e01",
   "metadata": {},
   "source": [
    "### "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}