sentiment_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\AppData\\Local\\Temp\\ipykernel_13388\\3770877507.py:1: DtypeWarning: Columns (12) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  data = pd.read_csv('data/combined_cleaned_data2/combined_with_fe.csv')\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv('data/combined_cleaned_data2/combined_with_fe.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['reviewText', 'review_clean', 'sentiment'], dtype='object')"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[['reviewText', 'review_clean', 'sentiment']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = data['reviewText'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "reviewText      object\n",
       "review_clean    object\n",
       "sentiment       object\n",
       "dtype: object"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(corpus[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73624"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_tokenize(sentence):\n",
    "  return sentence.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['tokenized'] = data['reviewText'].astype(str).apply(word_tokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['token_count'] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(data['tokenized'])):\n",
    "    data['token_count'][i] = len(data['tokenized'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewText</th>\n",
       "      <th>review_clean</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>tokenized</th>\n",
       "      <th>token_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>It diffuses a very mild light perfume, just wh...</td>\n",
       "      <td>wear mild perfume nice strong lotion day perfu...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[It, diffuses, a, very, mild, light, perfume,,...</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>All time favorite!!!! Wish they still carried ...</td>\n",
       "      <td>time still favorite carry wish</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[All, time, favorite!!!!, Wish, they, still, c...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>One of my wife's favorites.</td>\n",
       "      <td>one favorites wife's</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[One, of, my, wife's, favorites.]</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>If you have body acne this product is a must. ...</td>\n",
       "      <td>heal help within prevent already first wash mu...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[If, you, have, body, acne, this, product, is,...</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I really is what I expected</td>\n",
       "      <td>expect really</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[I, really, is, what, I, expected]</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73619</th>\n",
       "      <td>When I heard about Rail Simulator I was happy ...</td>\n",
       "      <td>think guide go acela maybe i've realize star t...</td>\n",
       "      <td>Negative</td>\n",
       "      <td>[When, I, heard, about, Rail, Simulator, I, wa...</td>\n",
       "      <td>1570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73620</th>\n",
       "      <td>very good</td>\n",
       "      <td>good</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[very, good]</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73621</th>\n",
       "      <td>does a great job at a great price.  Good quali...</td>\n",
       "      <td>great quality price good job</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[does, a, great, job, at, a, great, price., Go...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73622</th>\n",
       "      <td>Microsoft's point system doesn't really make a...</td>\n",
       "      <td>point else sense code point good make worked g...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[Microsoft's, point, system, doesn't, really, ...</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73623</th>\n",
       "      <td>I already had the game, but I was buying it ag...</td>\n",
       "      <td>reason buy buy something madden great already ...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[I, already, had, the, game,, but, I, was, buy...</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>73624 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              reviewText  \\\n",
       "0      It diffuses a very mild light perfume, just wh...   \n",
       "1      All time favorite!!!! Wish they still carried ...   \n",
       "2                            One of my wife's favorites.   \n",
       "3      If you have body acne this product is a must. ...   \n",
       "4                            I really is what I expected   \n",
       "...                                                  ...   \n",
       "73619  When I heard about Rail Simulator I was happy ...   \n",
       "73620                                          very good   \n",
       "73621  does a great job at a great price.  Good quali...   \n",
       "73622  Microsoft's point system doesn't really make a...   \n",
       "73623  I already had the game, but I was buying it ag...   \n",
       "\n",
       "                                            review_clean sentiment  \\\n",
       "0      wear mild perfume nice strong lotion day perfu...  Positive   \n",
       "1                         time still favorite carry wish  Positive   \n",
       "2                                   one favorites wife's  Positive   \n",
       "3      heal help within prevent already first wash mu...  Positive   \n",
       "4                                          expect really  Positive   \n",
       "...                                                  ...       ...   \n",
       "73619  think guide go acela maybe i've realize star t...  Negative   \n",
       "73620                                               good  Positive   \n",
       "73621                       great quality price good job  Positive   \n",
       "73622  point else sense code point good make worked g...  Positive   \n",
       "73623  reason buy buy something madden great already ...  Positive   \n",
       "\n",
       "                                               tokenized token_count  \n",
       "0      [It, diffuses, a, very, mild, light, perfume,,...          42  \n",
       "1      [All, time, favorite!!!!, Wish, they, still, c...           8  \n",
       "2                      [One, of, my, wife's, favorites.]           5  \n",
       "3      [If, you, have, body, acne, this, product, is,...          43  \n",
       "4                     [I, really, is, what, I, expected]           6  \n",
       "...                                                  ...         ...  \n",
       "73619  [When, I, heard, about, Rail, Simulator, I, wa...        1570  \n",
       "73620                                       [very, good]           2  \n",
       "73621  [does, a, great, job, at, a, great, price., Go...          11  \n",
       "73622  [Microsoft's, point, system, doesn't, really, ...          40  \n",
       "73623  [I, already, had, the, game,, but, I, was, buy...          69  \n",
       "\n",
       "[73624 rows x 5 columns]"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "reviewText      object\n",
       "review_clean    object\n",
       "sentiment       object\n",
       "tokenized       object\n",
       "token_count     object\n",
       "dtype: object"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          42\n",
       "1           8\n",
       "2           5\n",
       "3          43\n",
       "4           6\n",
       "         ... \n",
       "73619    1570\n",
       "73620       2\n",
       "73621      11\n",
       "73622      40\n",
       "73623      69\n",
       "Name: token_count, Length: 73624, dtype: int32"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.token_count.astype(int) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[data['token_count'] < 512]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(72643, 5)"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "308970\n",
      "54245\n"
     ]
    }
   ],
   "source": [
    "print(data[ data['sentiment'] == 'Positive'].size)\n",
    "print(data[ data['sentiment'] == 'Negative'].size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(72643, 5)"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\AppData\\Local\\Temp\\ipykernel_13388\\887699762.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data.review_clean = data.review_clean.astype(str)\n"
     ]
    }
   ],
   "source": [
    "data.review_clean = data.review_clean.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Admin\\anaconda3\\envs\\capstone3\\lib\\site-packages\\keras_preprocessing\\text.py:180: UserWarning: The `nb_words` argument in `Tokenizer` has been renamed `num_words`.\n",
      "  warnings.warn('The `nb_words` argument in `Tokenizer` '\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(nb_words=2000, lower=True,split=' ')\n",
    "tokenizer.fit_on_texts(data['review_clean'].values)\n",
    "# print(tokenizer.word_index)  # To see the dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tokenizer.texts_to_sequences(data['review_clean'].values)\n",
    "X = pad_sequences(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_3 (Embedding)     (None, 719, 128)          320000    \n",
      "                                                                 \n",
      " spatial_dropout1d_2 (Spatia  (None, 719, 128)         0         \n",
      " lDropout1D)                                                     \n",
      "                                                                 \n",
      " lstm_3 (LSTM)               (None, 196)               254800    \n",
      "                                                                 \n",
      " dense_3 (Dense)             (None, 2)                 394       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 575,194\n",
      "Trainable params: 575,194\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "embed_dim = 128\n",
    "lstm_out = 196\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(2500, embed_dim,input_length = X.shape[1]))\n",
    "model.add(SpatialDropout1D(0.4))\n",
    "model.add(LSTM(lstm_out, dropout=0.2))\n",
    "model.add(Dense(2,activation='softmax'))\n",
    "model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(48670, 178) (48670, 2)\n",
      "(23973, 178) (23973, 2)\n"
     ]
    }
   ],
   "source": [
    "Y = pd.get_dummies(data['sentiment']).values\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.33, random_state = 42)\n",
    "print(X_train.shape,Y_train.shape)\n",
    "print(X_test.shape,Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/7\n",
      "1542/1542 - 75s - loss: 0.3158 - accuracy: 0.8729 - 75s/epoch - 49ms/step\n",
      "Epoch 2/7\n",
      "1542/1542 - 74s - loss: 0.2627 - accuracy: 0.8929 - 74s/epoch - 48ms/step\n",
      "Epoch 3/7\n",
      "1542/1542 - 74s - loss: 0.2455 - accuracy: 0.9012 - 74s/epoch - 48ms/step\n",
      "Epoch 4/7\n",
      "1542/1542 - 78s - loss: 0.2311 - accuracy: 0.9081 - 78s/epoch - 50ms/step\n",
      "Epoch 5/7\n",
      "1542/1542 - 75s - loss: 0.2182 - accuracy: 0.9136 - 75s/epoch - 49ms/step\n",
      "Epoch 6/7\n",
      "1542/1542 - 76s - loss: 0.2073 - accuracy: 0.9193 - 76s/epoch - 49ms/step\n",
      "Epoch 7/7\n",
      "1542/1542 - 76s - loss: 0.1970 - accuracy: 0.9227 - 76s/epoch - 49ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1d571ec3040>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 32\n",
    "model.fit(X_train, Y_train, epochs = 7, batch_size=batch_size, verbose = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "713/713 - 11s - loss: 0.2946 - accuracy: 0.8867 - 11s/epoch - 16ms/step\n",
      "score: 0.29\n",
      "acc: 0.89\n"
     ]
    }
   ],
   "source": [
    "validation_size = 1500\n",
    "\n",
    "X_validate = X_test[-validation_size:]\n",
    "Y_validate = Y_test[-validation_size:]\n",
    "X_test = X_test[:-validation_size]\n",
    "Y_test = Y_test[:-validation_size]\n",
    "score,acc = model.evaluate(X_test, Y_test, verbose = 2, batch_size = batch_size)\n",
    "print(\"score: %.2f\" % (score))\n",
    "print(\"acc: %.2f\" % (acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix,classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix [[ 1605  2024]\n",
      " [  637 20030]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.44      0.55      3629\n",
      "           1       0.91      0.97      0.94     20667\n",
      "\n",
      "    accuracy                           0.89     24296\n",
      "   macro avg       0.81      0.71      0.74     24296\n",
      "weighted avg       0.88      0.89      0.88     24296\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Y_pred = model.predict(X_test,batch_size = batch_size)\n",
    "# predict_x=model.predict(X_test) \n",
    "Y_pred=np.argmax(Y_pred,axis=1)\n",
    "df_test = pd.DataFrame({'true': Y_test.tolist(), 'pred':Y_pred})\n",
    "df_test['true'] = df_test['true'].apply(lambda x: np.argmax(x))\n",
    "print(\"confusion matrix\",confusion_matrix(df_test.true, df_test.pred))\n",
    "print(classification_report(df_test.true, df_test.pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import resample\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate majority and minority classes\n",
    "data_majority = data[data['sentiment'] == 'Positive']\n",
    "data_minority = data[data['sentiment'] == 'Negative']\n",
    "\n",
    "bias = data_minority.shape[0]/data_majority.shape[0]\n",
    "# lets split train/test data first then \n",
    "train = pd.concat([data_majority.sample(frac=0.8,random_state=200),\n",
    "         data_minority.sample(frac=0.8,random_state=200)])\n",
    "test = pd.concat([data_majority.drop(data_majority.sample(frac=0.8,random_state=200).index),\n",
    "        data_minority.drop(data_minority.sample(frac=0.8,random_state=200).index)])\n",
    "\n",
    "train = shuffle(train)\n",
    "test = shuffle(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "positive data in training: 49435\n",
      "negative data in training: 8679\n",
      "positive data in test: 12359\n",
      "negative data in test: 2170\n"
     ]
    }
   ],
   "source": [
    "print('positive data in training:',(train.sentiment == 'Positive').sum())\n",
    "print('negative data in training:',(train.sentiment == 'Negative').sum())\n",
    "print('positive data in test:',(test.sentiment == 'Positive').sum())\n",
    "print('negative data in test:',(test.sentiment == 'Negative').sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "positive data in training: 49435\n",
      "negative data in training: 8679\n",
      "positive data in test: 12359\n",
      "negative data in test: 2170\n"
     ]
    }
   ],
   "source": [
    "print('positive data in training:',(train.sentiment == 'Positive').sum())\n",
    "print('negative data in training:',(train.sentiment == 'Negative').sum())\n",
    "print('positive data in test:',(test.sentiment == 'Positive').sum())\n",
    "print('negative data in test:',(test.sentiment == 'Negative').sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "majority class before upsample: (49435, 5)\n",
      "minority class before upsample: (8679, 5)\n",
      "After upsampling\n",
      "Positive    49435\n",
      "Negative    49435\n",
      "Name: sentiment, dtype: int64\n",
      "x_train shape: (98870, 186)\n",
      "x_test shape (14529, 186)\n"
     ]
    }
   ],
   "source": [
    "# Separate majority and minority classes in training data for upsampling \n",
    "data_majority = train[train['sentiment'] == 'Positive']\n",
    "data_minority = train[train['sentiment'] == 'Negative']\n",
    "\n",
    "print(\"majority class before upsample:\",data_majority.shape)\n",
    "print(\"minority class before upsample:\",data_minority.shape)\n",
    "\n",
    "# Upsample minority class\n",
    "data_minority_upsampled = resample(data_minority, \n",
    "                                 replace=True,     # sample with replacement\n",
    "                                 n_samples= data_majority.shape[0],    # to match majority class\n",
    "                                 random_state=123) # reproducible results\n",
    " \n",
    "# Combine majority class with upsampled minority class\n",
    "data_upsampled = pd.concat([data_majority, data_minority_upsampled])\n",
    " \n",
    "# Display new class counts\n",
    "print(\"After upsampling\\n\",data_upsampled.sentiment.value_counts(),sep = \"\")\n",
    "\n",
    "max_fatures = 2000\n",
    "tokenizer = Tokenizer(num_words=max_fatures, split=' ')\n",
    "tokenizer.fit_on_texts(data['review_clean'].values) # training with whole data\n",
    "\n",
    "X_train = tokenizer.texts_to_sequences(data_upsampled['review_clean'].values)\n",
    "X_train = pad_sequences(X_train,maxlen=186)\n",
    "Y_train = pd.get_dummies(data_upsampled['sentiment']).values\n",
    "print('x_train shape:',X_train.shape)\n",
    "\n",
    "X_test = tokenizer.texts_to_sequences(test['review_clean'].values)\n",
    "X_test = pad_sequences(X_test,maxlen=186)\n",
    "Y_test = pd.get_dummies(test['sentiment']).values\n",
    "print(\"x_test shape\", X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_9\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_7 (Embedding)     (None, 186, 128)          320000    \n",
      "                                                                 \n",
      " spatial_dropout1d_6 (Spatia  (None, 186, 128)         0         \n",
      " lDropout1D)                                                     \n",
      "                                                                 \n",
      " lstm_7 (LSTM)               (None, 196)               254800    \n",
      "                                                                 \n",
      " dense_7 (Dense)             (None, 2)                 394       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 575,194\n",
      "Trainable params: 575,194\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "embed_dim = 128\n",
    "lstm_out = 196\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(2500, embed_dim,input_length = X.shape[1]))\n",
    "model.add(SpatialDropout1D(0.4))\n",
    "model.add(LSTM(lstm_out, dropout=0.2))\n",
    "model.add(Dense(2,activation='softmax'))\n",
    "model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.config.run_functions_eagerly(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Admin\\anaconda3\\envs\\capstone3\\lib\\site-packages\\tensorflow\\python\\data\\ops\\structured_function.py:264: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "773/773 [==============================] - 32s 41ms/step - loss: 0.9497 - accuracy: 0.6847\n",
      "Epoch 2/10\n",
      "773/773 [==============================] - 32s 41ms/step - loss: 0.8042 - accuracy: 0.7511\n",
      "Epoch 3/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.7483 - accuracy: 0.7697\n",
      "Epoch 4/10\n",
      "773/773 [==============================] - 33s 42ms/step - loss: 0.6950 - accuracy: 0.7888\n",
      "Epoch 5/10\n",
      "773/773 [==============================] - 33s 42ms/step - loss: 0.6447 - accuracy: 0.8078\n",
      "Epoch 6/10\n",
      "773/773 [==============================] - 33s 42ms/step - loss: 0.5973 - accuracy: 0.8241\n",
      "Epoch 7/10\n",
      "773/773 [==============================] - 33s 42ms/step - loss: 0.5560 - accuracy: 0.8378\n",
      "Epoch 8/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.5117 - accuracy: 0.8533\n",
      "Epoch 9/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.4690 - accuracy: 0.8667\n",
      "Epoch 10/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.4351 - accuracy: 0.8777\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1d4e543f370>"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 128\n",
    "# also adding weights\n",
    "class_weights = {0: 1 ,\n",
    "                1: 1/bias }\n",
    "model.fit(X_train, Y_train, epochs = 10, batch_size=batch_size, verbose = 1,\n",
    "          class_weight=class_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Admin\\anaconda3\\envs\\capstone3\\lib\\site-packages\\tensorflow\\python\\data\\ops\\structured_function.py:264: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "773/773 [==============================] - 33s 42ms/step - loss: 0.3933 - accuracy: 0.8926\n",
      "Epoch 2/10\n",
      "773/773 [==============================] - 33s 42ms/step - loss: 0.3603 - accuracy: 0.9008\n",
      "Epoch 3/10\n",
      "773/773 [==============================] - 33s 42ms/step - loss: 0.3349 - accuracy: 0.9090\n",
      "Epoch 4/10\n",
      "773/773 [==============================] - 33s 42ms/step - loss: 0.3130 - accuracy: 0.9148\n",
      "Epoch 5/10\n",
      "773/773 [==============================] - 33s 42ms/step - loss: 0.2997 - accuracy: 0.9200\n",
      "Epoch 6/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.2807 - accuracy: 0.9246\n",
      "Epoch 7/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.2688 - accuracy: 0.9284\n",
      "Epoch 8/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.2536 - accuracy: 0.9320\n",
      "Epoch 9/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.2414 - accuracy: 0.9344\n",
      "Epoch 10/10\n",
      "773/773 [==============================] - 34s 43ms/step - loss: 0.2349 - accuracy: 0.9379\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1d4f7dcbd60>"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, Y_train, epochs = 10, batch_size=batch_size, verbose = 1,\n",
    "          class_weight=class_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix [[ 1089  1081]\n",
      " [  712 11647]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.60      0.50      0.55      2170\n",
      "           1       0.92      0.94      0.93     12359\n",
      "\n",
      "    accuracy                           0.88     14529\n",
      "   macro avg       0.76      0.72      0.74     14529\n",
      "weighted avg       0.87      0.88      0.87     14529\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Y_pred = model.predict(X_test,batch_size = batch_size)\n",
    "# predict_x=model.predict(X_test) \n",
    "Y_pred=np.argmax(Y_pred,axis=1)\n",
    "df_test = pd.DataFrame({'true': Y_test.tolist(), 'pred':Y_pred})\n",
    "df_test['true'] = df_test['true'].apply(lambda x: np.argmax(x))\n",
    "print(\"confusion matrix\",confusion_matrix(df_test.true, df_test.pred))\n",
    "print(classification_report(df_test.true, df_test.pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('model10.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0 1399\n",
      "    37 1576  123   37]]\n",
      "1/1 - 0s - 28ms/epoch - 28ms/step\n",
      "positive\n"
     ]
    }
   ],
   "source": [
    "twt = ['this thing has no bad thing']\n",
    "#vectorizing the tweet by the pre-fitted tokenizer instance\n",
    "twt = tokenizer.texts_to_sequences(twt)\n",
    "#padding the tweet to have exactly the same shape as `embedding_2` input\n",
    "twt = pad_sequences(twt, maxlen=186, dtype='int32', value=0)\n",
    "print(twt)\n",
    "sentiment = model.predict(twt,batch_size=1,verbose = 2)[0]\n",
    "if(np.argmax(sentiment) == 0):\n",
    "    print(\"negative\")\n",
    "elif (np.argmax(sentiment) == 1):\n",
    "    print(\"positive\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = pd.get_dummies(data['sentiment']).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1],\n",
       "       [0, 1],\n",
       "       [0, 1],\n",
       "       ...,\n",
       "       [0, 1],\n",
       "       [0, 1],\n",
       "       [0, 1]], dtype=uint8)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Admin\\anaconda3\\envs\\capstone3\\lib\\site-packages\\tensorflow\\python\\data\\ops\\structured_function.py:264: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score: 0.59\n",
      "acc: 0.88\n"
     ]
    }
   ],
   "source": [
    "validation_size = 1500\n",
    "\n",
    "X_validate = X_test[-validation_size:]\n",
    "Y_validate = Y_test[-validation_size:]\n",
    "X_test = X_test[:-validation_size]\n",
    "Y_test = Y_test[:-validation_size]\n",
    "score,acc = model.evaluate(X_test, Y_test, verbose = 0, batch_size = batch_size)\n",
    "print(\"score: %.2f\" % (score))\n",
    "print(\"acc: %.2f\" % (acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pos_acc 94.95341614906833 %\n",
      "neg_acc 51.886792452830186 %\n"
     ]
    }
   ],
   "source": [
    "pos_cnt, neg_cnt, pos_correct, neg_correct = 0, 0, 0, 0\n",
    "for x in range(len(X_validate)):\n",
    "    \n",
    "    result = model.predict(X_validate[x].reshape(1,X_test.shape[1]),batch_size=1,verbose = 0)[0]\n",
    "   \n",
    "    if np.argmax(result) == np.argmax(Y_validate[x]):\n",
    "        if np.argmax(Y_validate[x]) == 0:\n",
    "            neg_correct += 1\n",
    "        else:\n",
    "            pos_correct += 1\n",
    "       \n",
    "    if np.argmax(Y_validate[x]) == 0:\n",
    "        neg_cnt += 1\n",
    "    else:\n",
    "        pos_cnt += 1\n",
    "\n",
    "\n",
    "\n",
    "print(\"pos_acc\", pos_correct/pos_cnt*100, \"%\")\n",
    "print(\"neg_acc\", neg_correct/neg_cnt*100, \"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_10\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_8 (Embedding)     (None, 178, 128)          256000    \n",
      "                                                                 \n",
      " spatial_dropout1d_7 (Spatia  (None, 178, 128)         0         \n",
      " lDropout1D)                                                     \n",
      "                                                                 \n",
      " lstm_8 (LSTM)               (None, 196)               254800    \n",
      "                                                                 \n",
      " dense_8 (Dense)             (None, 2)                 394       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 511,194\n",
      "Trainable params: 511,194\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "embed_dim = 128\n",
    "lstm_out = 196\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(2000, embed_dim,input_length = X.shape[1]))\n",
    "model.add(SpatialDropout1D(0.4))\n",
    "model.add(LSTM(lstm_out, dropout=0.2))\n",
    "model.add(Dense(2,activation='softmax'))\n",
    "model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate majority and minority classes\n",
    "data_majority = data[data['sentiment'] == 'Positive']\n",
    "data_minority = data[data['sentiment'] == 'Negative']\n",
    "\n",
    "bias = data_minority.shape[0]/data_majority.shape[0]\n",
    "# lets split train/test data first then \n",
    "train = pd.concat([data_majority.sample(frac=0.8,random_state=200),\n",
    "         data_minority.sample(frac=0.8,random_state=200)])\n",
    "test = pd.concat([data_majority.drop(data_majority.sample(frac=0.8,random_state=200).index),\n",
    "        data_minority.drop(data_minority.sample(frac=0.8,random_state=200).index)])\n",
    "\n",
    "train = shuffle(train)\n",
    "test = shuffle(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "positive data in training: 49435\n",
      "negative data in training: 8679\n",
      "positive data in test: 12359\n",
      "negative data in test: 2170\n"
     ]
    }
   ],
   "source": [
    "print('positive data in training:',(train.sentiment == 'Positive').sum())\n",
    "print('negative data in training:',(train.sentiment == 'Negative').sum())\n",
    "print('positive data in test:',(test.sentiment == 'Positive').sum())\n",
    "print('negative data in test:',(test.sentiment == 'Negative').sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "majority class before upsample: (49435, 5)\n",
      "minority class before upsample: (8679, 5)\n",
      "After upsampling\n",
      "Positive    49435\n",
      "Negative    49435\n",
      "Name: sentiment, dtype: int64\n",
      "x_train shape: (98870, 178)\n",
      "x_test shape (14529, 178)\n"
     ]
    }
   ],
   "source": [
    "# Separate majority and minority classes in training data for upsampling \n",
    "data_majority = train[train['sentiment'] == 'Positive']\n",
    "data_minority = train[train['sentiment'] == 'Negative']\n",
    "\n",
    "print(\"majority class before upsample:\",data_majority.shape)\n",
    "print(\"minority class before upsample:\",data_minority.shape)\n",
    "\n",
    "# Upsample minority class\n",
    "data_minority_upsampled = resample(data_minority, \n",
    "                                 replace=True,     # sample with replacement\n",
    "                                 n_samples= data_majority.shape[0],    # to match majority class\n",
    "                                 random_state=123) # reproducible results\n",
    " \n",
    "# Combine majority class with upsampled minority class\n",
    "data_upsampled = pd.concat([data_majority, data_minority_upsampled])\n",
    " \n",
    "# Display new class counts\n",
    "print(\"After upsampling\\n\",data_upsampled.sentiment.value_counts(),sep = \"\")\n",
    "\n",
    "max_fatures = 2000\n",
    "tokenizer = Tokenizer(num_words=max_fatures, split=' ')\n",
    "tokenizer.fit_on_texts(data['review_clean'].values) # training with whole data\n",
    "\n",
    "X_train = tokenizer.texts_to_sequences(data_upsampled['review_clean'].values)\n",
    "X_train = pad_sequences(X_train,maxlen=178)\n",
    "Y_train = pd.get_dummies(data_upsampled['sentiment']).values\n",
    "print('x_train shape:',X_train.shape)\n",
    "\n",
    "X_test = tokenizer.texts_to_sequences(test['review_clean'].values)\n",
    "X_test = pad_sequences(X_test,maxlen=178)\n",
    "Y_test = pd.get_dummies(test['sentiment']).values\n",
    "print(\"x_test shape\", X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Admin\\anaconda3\\envs\\capstone3\\lib\\site-packages\\tensorflow\\python\\data\\ops\\structured_function.py:264: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "773/773 [==============================] - 32s 41ms/step - loss: 0.9582 - accuracy: 0.6788\n",
      "Epoch 2/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.8021 - accuracy: 0.7495\n",
      "Epoch 3/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.7489 - accuracy: 0.7704\n",
      "Epoch 4/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.7031 - accuracy: 0.7858\n",
      "Epoch 5/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.6594 - accuracy: 0.8022\n",
      "Epoch 6/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.6116 - accuracy: 0.8207\n",
      "Epoch 7/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.5657 - accuracy: 0.8365\n",
      "Epoch 8/10\n",
      "773/773 [==============================] - 33s 42ms/step - loss: 0.5268 - accuracy: 0.8499\n",
      "Epoch 9/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.4898 - accuracy: 0.8622\n",
      "Epoch 10/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.4485 - accuracy: 0.8756\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "# also adding weights\n",
    "class_weights = {0: 1 ,\n",
    "                1: 1/bias }\n",
    "history = model.fit(X_train, Y_train, epochs = 10, batch_size=batch_size, verbose = 1,\n",
    "          class_weight=class_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "773/773 [==============================] - 32s 41ms/step - loss: 0.4215 - accuracy: 0.8837\n",
      "Epoch 2/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.3973 - accuracy: 0.8909\n",
      "Epoch 3/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.3628 - accuracy: 0.9017\n",
      "Epoch 4/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.3383 - accuracy: 0.9098\n",
      "Epoch 5/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.3136 - accuracy: 0.9168\n",
      "Epoch 6/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.2963 - accuracy: 0.9208\n",
      "Epoch 7/10\n",
      "773/773 [==============================] - 32s 42ms/step - loss: 0.2790 - accuracy: 0.9264\n",
      "Epoch 8/10\n",
      "773/773 [==============================] - 33s 42ms/step - loss: 0.2608 - accuracy: 0.9303\n",
      "Epoch 9/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.2485 - accuracy: 0.9334\n",
      "Epoch 10/10\n",
      "773/773 [==============================] - 33s 43ms/step - loss: 0.2387 - accuracy: 0.9370\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "# also adding weights\n",
    "class_weights = {0: 1 ,\n",
    "                1: 1/bias }\n",
    "history = model.fit(X_train, Y_train, epochs = 10, batch_size=batch_size, verbose = 1,\n",
    "          class_weight=class_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix [[ 1018  1152]\n",
      " [  630 11729]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.62      0.47      0.53      2170\n",
      "           1       0.91      0.95      0.93     12359\n",
      "\n",
      "    accuracy                           0.88     14529\n",
      "   macro avg       0.76      0.71      0.73     14529\n",
      "weighted avg       0.87      0.88      0.87     14529\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Y_pred = model.predict(X_test,batch_size = batch_size)\n",
    "# predict_x=model.predict(X_test) \n",
    "Y_pred=np.argmax(Y_pred,axis=1)\n",
    "df_test = pd.DataFrame({'true': Y_test.tolist(), 'pred':Y_pred})\n",
    "df_test['true'] = df_test['true'].apply(lambda x: np.argmax(x))\n",
    "print(\"confusion matrix\",confusion_matrix(df_test.true, df_test.pred))\n",
    "print(classification_report(df_test.true, df_test.pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('model11.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0  733  686  431  277  561  344 1012]]\n",
      "1/1 - 0s - 26ms/epoch - 26ms/step\n",
      "positive\n"
     ]
    }
   ],
   "source": [
    "twt = ['It cools the room very quickly and produces very smooth noise']\n",
    "#vectorizing the tweet by the pre-fitted tokenizer instance\n",
    "twt = tokenizer.texts_to_sequences(twt)\n",
    "#padding the tweet to have exactly the same shape as `embedding_2` input\n",
    "twt = pad_sequences(twt, maxlen=178, dtype='int32', value=0)\n",
    "print(twt)\n",
    "sentiment = model.predict(twt,batch_size=1,verbose = 2)[0]\n",
    "if(np.argmax(sentiment) == 0):\n",
    "    print(\"negative\")\n",
    "elif (np.argmax(sentiment) == 1):\n",
    "    print(\"positive\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score: 0.62\n",
      "acc: 0.88\n"
     ]
    }
   ],
   "source": [
    "validation_size = 1500\n",
    "\n",
    "X_validate = X_test[-validation_size:]\n",
    "Y_validate = Y_test[-validation_size:]\n",
    "X_test = X_test[:-validation_size]\n",
    "Y_test = Y_test[:-validation_size]\n",
    "score,acc = model.evaluate(X_test, Y_test, verbose = 0, batch_size = batch_size)\n",
    "print(\"score: %.2f\" % (score))\n",
    "print(\"acc: %.2f\" % (acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pos_acc 96.18677042801556 %\n",
      "neg_acc 49.76744186046512 %\n"
     ]
    }
   ],
   "source": [
    "pos_cnt, neg_cnt, pos_correct, neg_correct = 0, 0, 0, 0\n",
    "for x in range(len(X_validate)):\n",
    "    \n",
    "    result = model.predict(X_validate[x].reshape(1,X_test.shape[1]),batch_size=1,verbose = 0)[0]\n",
    "   \n",
    "    if np.argmax(result) == np.argmax(Y_validate[x]):\n",
    "        if np.argmax(Y_validate[x]) == 0:\n",
    "            neg_correct += 1\n",
    "        else:\n",
    "            pos_correct += 1\n",
    "       \n",
    "    if np.argmax(Y_validate[x]) == 0:\n",
    "        neg_cnt += 1\n",
    "    else:\n",
    "        pos_cnt += 1\n",
    "\n",
    "\n",
    "\n",
    "print(\"pos_acc\", pos_correct/pos_cnt*100, \"%\")\n",
    "print(\"neg_acc\", neg_correct/neg_cnt*100, \"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "afcb804707279b9b642cf2a94c26a02d7fe35902e873945e34f9c8bae120ddc3"
  },
  "kernelspec": {
   "display_name": "Python 3.8.0 ('capstone3')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}