clustering-sample.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pos = pd.read_csv('data/combined_cleaned_data/final_pos.csv')\n",
    "df_neg = pd.read_csv('data/combined_cleaned_data/final_neg.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\AppData\\Local\\Temp\\ipykernel_10108\\105442801.py:1: DtypeWarning: Columns (12) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv('data/combined_cleaned_data/combined_with_fe.csv')\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('data/combined_cleaned_data/combined_with_fe.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method IndexOpsMixin.nunique of 0        B0009RF9DW\n",
       "1        B0012Y0ZG2\n",
       "2        B0012Y0ZG2\n",
       "3        B000FI4S1E\n",
       "4        B0012Y0ZG2\n",
       "            ...    \n",
       "73619    B000YIYB3K\n",
       "73620    B003VJPMT6\n",
       "73621    B00FWK8UWI\n",
       "73622    B000B9RI14\n",
       "73623    B006VB2UNM\n",
       "Name: asin, Length: 73624, dtype: object>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.asin.nunique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(73624, 16)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0.1', 'Unnamed: 0', 'overall', 'verified', 'reviewTime',\n",
       "       'reviewerID', 'asin', 'style', 'reviewerName', 'reviewText', 'summary',\n",
       "       'unixReviewTime', 'vote', 'image', 'review_clean', 'sentiment'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(df.columns[[0, 1]], axis = 1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(subset=['reviewText', 'review_clean', 'summary'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>overall</th>\n",
       "      <th>verified</th>\n",
       "      <th>reviewTime</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>style</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>unixReviewTime</th>\n",
       "      <th>vote</th>\n",
       "      <th>image</th>\n",
       "      <th>review_clean</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>09 24, 2013</td>\n",
       "      <td>A3CZ890UHC8HHZ</td>\n",
       "      <td>B0009RF9DW</td>\n",
       "      <td>{'Size:': ' 6.6 oz'}</td>\n",
       "      <td>Amazon Customer</td>\n",
       "      <td>It diffuses a very mild light perfume, just wh...</td>\n",
       "      <td>What I expected</td>\n",
       "      <td>1379980800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>wear mild perfume nice strong lotion day perfu...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>07 12, 2014</td>\n",
       "      <td>A125TMC44CJBKK</td>\n",
       "      <td>B0012Y0ZG2</td>\n",
       "      <td>{'Size:': ' B-020'}</td>\n",
       "      <td>Mustang Shelly</td>\n",
       "      <td>All time favorite!!!! Wish they still carried ...</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>1405123200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>time still favorite carry wish</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>12 28, 2014</td>\n",
       "      <td>A3JDSBP01M7R2R</td>\n",
       "      <td>B0012Y0ZG2</td>\n",
       "      <td>{'Size:': ' 112'}</td>\n",
       "      <td>Tim A. Brooks</td>\n",
       "      <td>One of my wife's favorites.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>1419724800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>one favorites wife's</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>03 3, 2014</td>\n",
       "      <td>A2HT5WENL8E5S6</td>\n",
       "      <td>B000FI4S1E</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Shasta Masser</td>\n",
       "      <td>If you have body acne this product is a must. ...</td>\n",
       "      <td>Acne Body Wash A MUST</td>\n",
       "      <td>1393804800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>heal help within prevent already first wash mu...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>11 7, 2014</td>\n",
       "      <td>A7LAYX9R7IMOO</td>\n",
       "      <td>B0012Y0ZG2</td>\n",
       "      <td>{'Size:': ' 123'}</td>\n",
       "      <td>andre luiz</td>\n",
       "      <td>I really is what I expected</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>1415318400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>expect really</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  overall  verified   reviewTime      reviewerID        asin  \\\n",
       "0      0        5      True  09 24, 2013  A3CZ890UHC8HHZ  B0009RF9DW   \n",
       "1      1        5     False  07 12, 2014  A125TMC44CJBKK  B0012Y0ZG2   \n",
       "2      2        5      True  12 28, 2014  A3JDSBP01M7R2R  B0012Y0ZG2   \n",
       "3      3        5      True   03 3, 2014  A2HT5WENL8E5S6  B000FI4S1E   \n",
       "4      4        5     False   11 7, 2014   A7LAYX9R7IMOO  B0012Y0ZG2   \n",
       "\n",
       "                  style     reviewerName  \\\n",
       "0  {'Size:': ' 6.6 oz'}  Amazon Customer   \n",
       "1   {'Size:': ' B-020'}   Mustang Shelly   \n",
       "2     {'Size:': ' 112'}    Tim A. Brooks   \n",
       "3                   NaN    Shasta Masser   \n",
       "4     {'Size:': ' 123'}       andre luiz   \n",
       "\n",
       "                                          reviewText                summary  \\\n",
       "0  It diffuses a very mild light perfume, just wh...        What I expected   \n",
       "1  All time favorite!!!! Wish they still carried ...             Five Stars   \n",
       "2                        One of my wife's favorites.             Five Stars   \n",
       "3  If you have body acne this product is a must. ...  Acne Body Wash A MUST   \n",
       "4                        I really is what I expected             Five Stars   \n",
       "\n",
       "   unixReviewTime vote image  \\\n",
       "0      1379980800  NaN   NaN   \n",
       "1      1405123200  NaN   NaN   \n",
       "2      1419724800  NaN   NaN   \n",
       "3      1393804800  NaN   NaN   \n",
       "4      1415318400  NaN   NaN   \n",
       "\n",
       "                                        review_clean sentiment  \n",
       "0  wear mild perfume nice strong lotion day perfu...  Positive  \n",
       "1                     time still favorite carry wish  Positive  \n",
       "2                               one favorites wife's  Positive  \n",
       "3  heal help within prevent already first wash mu...  Positive  \n",
       "4                                      expect really  Positive  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop('index', axis='columns', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(['reviewTime', 'asin', 'style', 'reviewerName', 'unixReviewTime', 'vote', 'image'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall</th>\n",
       "      <th>verified</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>review_clean</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A3CZ890UHC8HHZ</td>\n",
       "      <td>It diffuses a very mild light perfume, just wh...</td>\n",
       "      <td>What I expected</td>\n",
       "      <td>wear mild perfume nice strong lotion day perfu...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>A125TMC44CJBKK</td>\n",
       "      <td>All time favorite!!!! Wish they still carried ...</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>time still favorite carry wish</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A3JDSBP01M7R2R</td>\n",
       "      <td>One of my wife's favorites.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>one favorites wife's</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A2HT5WENL8E5S6</td>\n",
       "      <td>If you have body acne this product is a must. ...</td>\n",
       "      <td>Acne Body Wash A MUST</td>\n",
       "      <td>heal help within prevent already first wash mu...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>A7LAYX9R7IMOO</td>\n",
       "      <td>I really is what I expected</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>expect really</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   overall  verified      reviewerID  \\\n",
       "0        5      True  A3CZ890UHC8HHZ   \n",
       "1        5     False  A125TMC44CJBKK   \n",
       "2        5      True  A3JDSBP01M7R2R   \n",
       "3        5      True  A2HT5WENL8E5S6   \n",
       "4        5     False   A7LAYX9R7IMOO   \n",
       "\n",
       "                                          reviewText                summary  \\\n",
       "0  It diffuses a very mild light perfume, just wh...        What I expected   \n",
       "1  All time favorite!!!! Wish they still carried ...             Five Stars   \n",
       "2                        One of my wife's favorites.             Five Stars   \n",
       "3  If you have body acne this product is a must. ...  Acne Body Wash A MUST   \n",
       "4                        I really is what I expected             Five Stars   \n",
       "\n",
       "                                        review_clean sentiment  \n",
       "0  wear mild perfume nice strong lotion day perfu...  Positive  \n",
       "1                     time still favorite carry wish  Positive  \n",
       "2                               one favorites wife's  Positive  \n",
       "3  heal help within prevent already first wash mu...  Positive  \n",
       "4                                      expect really  Positive  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "lb=LabelEncoder()\n",
    "df['sentiment'] = lb.fit_transform(df['sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall</th>\n",
       "      <th>verified</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>review_clean</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A3CZ890UHC8HHZ</td>\n",
       "      <td>It diffuses a very mild light perfume, just wh...</td>\n",
       "      <td>What I expected</td>\n",
       "      <td>wear mild perfume nice strong lotion day perfu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>A125TMC44CJBKK</td>\n",
       "      <td>All time favorite!!!! Wish they still carried ...</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>time still favorite carry wish</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A3JDSBP01M7R2R</td>\n",
       "      <td>One of my wife's favorites.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>one favorites wife's</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A2HT5WENL8E5S6</td>\n",
       "      <td>If you have body acne this product is a must. ...</td>\n",
       "      <td>Acne Body Wash A MUST</td>\n",
       "      <td>heal help within prevent already first wash mu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>A7LAYX9R7IMOO</td>\n",
       "      <td>I really is what I expected</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>expect really</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73363</th>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>A1MB5HUA3WYEJC</td>\n",
       "      <td>When I heard about Rail Simulator I was happy ...</td>\n",
       "      <td>So much promise, so many times being delayed, ...</td>\n",
       "      <td>think guide go acela maybe i've realize star t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73364</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A2J4588ZHYJORN</td>\n",
       "      <td>very good</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>good</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73365</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A2P06K653ZTGXN</td>\n",
       "      <td>does a great job at a great price.  Good quali...</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>great quality price good job</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73366</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A1S8HP3G8DTMFH</td>\n",
       "      <td>Microsoft's point system doesn't really make a...</td>\n",
       "      <td>The points are weird</td>\n",
       "      <td>point else sense code point good make worked g...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73367</th>\n",
       "      <td>4</td>\n",
       "      <td>True</td>\n",
       "      <td>A1KYXIBRJS2JE9</td>\n",
       "      <td>I already had the game, but I was buying it ag...</td>\n",
       "      <td>Replacement</td>\n",
       "      <td>reason buy buy something madden great already ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>73368 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       overall  verified      reviewerID  \\\n",
       "0            5      True  A3CZ890UHC8HHZ   \n",
       "1            5     False  A125TMC44CJBKK   \n",
       "2            5      True  A3JDSBP01M7R2R   \n",
       "3            5      True  A2HT5WENL8E5S6   \n",
       "4            5     False   A7LAYX9R7IMOO   \n",
       "...        ...       ...             ...   \n",
       "73363        2     False  A1MB5HUA3WYEJC   \n",
       "73364        5      True  A2J4588ZHYJORN   \n",
       "73365        5      True  A2P06K653ZTGXN   \n",
       "73366        5      True  A1S8HP3G8DTMFH   \n",
       "73367        4      True  A1KYXIBRJS2JE9   \n",
       "\n",
       "                                              reviewText  \\\n",
       "0      It diffuses a very mild light perfume, just wh...   \n",
       "1      All time favorite!!!! Wish they still carried ...   \n",
       "2                            One of my wife's favorites.   \n",
       "3      If you have body acne this product is a must. ...   \n",
       "4                            I really is what I expected   \n",
       "...                                                  ...   \n",
       "73363  When I heard about Rail Simulator I was happy ...   \n",
       "73364                                          very good   \n",
       "73365  does a great job at a great price.  Good quali...   \n",
       "73366  Microsoft's point system doesn't really make a...   \n",
       "73367  I already had the game, but I was buying it ag...   \n",
       "\n",
       "                                                 summary  \\\n",
       "0                                        What I expected   \n",
       "1                                             Five Stars   \n",
       "2                                             Five Stars   \n",
       "3                                  Acne Body Wash A MUST   \n",
       "4                                             Five Stars   \n",
       "...                                                  ...   \n",
       "73363  So much promise, so many times being delayed, ...   \n",
       "73364                                         Five Stars   \n",
       "73365                                         Five Stars   \n",
       "73366                               The points are weird   \n",
       "73367                                        Replacement   \n",
       "\n",
       "                                            review_clean  sentiment  \n",
       "0      wear mild perfume nice strong lotion day perfu...          1  \n",
       "1                         time still favorite carry wish          1  \n",
       "2                                   one favorites wife's          1  \n",
       "3      heal help within prevent already first wash mu...          1  \n",
       "4                                          expect really          1  \n",
       "...                                                  ...        ...  \n",
       "73363  think guide go acela maybe i've realize star t...          0  \n",
       "73364                                               good          1  \n",
       "73365                       great quality price good job          1  \n",
       "73366  point else sense code point good make worked g...          1  \n",
       "73367  reason buy buy something madden great already ...          1  \n",
       "\n",
       "[73368 rows x 7 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.loc[df.verified, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall</th>\n",
       "      <th>verified</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>review_clean</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A3CZ890UHC8HHZ</td>\n",
       "      <td>It diffuses a very mild light perfume, just wh...</td>\n",
       "      <td>What I expected</td>\n",
       "      <td>wear mild perfume nice strong lotion day perfu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A3JDSBP01M7R2R</td>\n",
       "      <td>One of my wife's favorites.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>one favorites wife's</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A2HT5WENL8E5S6</td>\n",
       "      <td>If you have body acne this product is a must. ...</td>\n",
       "      <td>Acne Body Wash A MUST</td>\n",
       "      <td>heal help within prevent already first wash mu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A29PKN4LL05QGF</td>\n",
       "      <td>Lathers well and smells great.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>well great lather smell</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>AD42QCRPG1RKG</td>\n",
       "      <td>It was good, my loved her gift!</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>love good gift</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73361</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A2RP3NEF3K3627</td>\n",
       "      <td>Perfect</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>perfect</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73364</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A2J4588ZHYJORN</td>\n",
       "      <td>very good</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>good</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73365</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A2P06K653ZTGXN</td>\n",
       "      <td>does a great job at a great price.  Good quali...</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>great quality price good job</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73366</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>A1S8HP3G8DTMFH</td>\n",
       "      <td>Microsoft's point system doesn't really make a...</td>\n",
       "      <td>The points are weird</td>\n",
       "      <td>point else sense code point good make worked g...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73367</th>\n",
       "      <td>4</td>\n",
       "      <td>True</td>\n",
       "      <td>A1KYXIBRJS2JE9</td>\n",
       "      <td>I already had the game, but I was buying it ag...</td>\n",
       "      <td>Replacement</td>\n",
       "      <td>reason buy buy something madden great already ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>60182 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       overall  verified      reviewerID  \\\n",
       "0            5      True  A3CZ890UHC8HHZ   \n",
       "2            5      True  A3JDSBP01M7R2R   \n",
       "3            5      True  A2HT5WENL8E5S6   \n",
       "5            5      True  A29PKN4LL05QGF   \n",
       "8            5      True   AD42QCRPG1RKG   \n",
       "...        ...       ...             ...   \n",
       "73361        5      True  A2RP3NEF3K3627   \n",
       "73364        5      True  A2J4588ZHYJORN   \n",
       "73365        5      True  A2P06K653ZTGXN   \n",
       "73366        5      True  A1S8HP3G8DTMFH   \n",
       "73367        4      True  A1KYXIBRJS2JE9   \n",
       "\n",
       "                                              reviewText  \\\n",
       "0      It diffuses a very mild light perfume, just wh...   \n",
       "2                            One of my wife's favorites.   \n",
       "3      If you have body acne this product is a must. ...   \n",
       "5                         Lathers well and smells great.   \n",
       "8                        It was good, my loved her gift!   \n",
       "...                                                  ...   \n",
       "73361                                            Perfect   \n",
       "73364                                          very good   \n",
       "73365  does a great job at a great price.  Good quali...   \n",
       "73366  Microsoft's point system doesn't really make a...   \n",
       "73367  I already had the game, but I was buying it ag...   \n",
       "\n",
       "                     summary  \\\n",
       "0            What I expected   \n",
       "2                 Five Stars   \n",
       "3      Acne Body Wash A MUST   \n",
       "5                 Five Stars   \n",
       "8                 Five Stars   \n",
       "...                      ...   \n",
       "73361             Five Stars   \n",
       "73364             Five Stars   \n",
       "73365             Five Stars   \n",
       "73366   The points are weird   \n",
       "73367            Replacement   \n",
       "\n",
       "                                            review_clean  sentiment  \n",
       "0      wear mild perfume nice strong lotion day perfu...          1  \n",
       "2                                   one favorites wife's          1  \n",
       "3      heal help within prevent already first wash mu...          1  \n",
       "5                                well great lather smell          1  \n",
       "8                                         love good gift          1  \n",
       "...                                                  ...        ...  \n",
       "73361                                            perfect          1  \n",
       "73364                                               good          1  \n",
       "73365                       great quality price good job          1  \n",
       "73366  point else sense code point good make worked g...          1  \n",
       "73367  reason buy buy something madden great already ...          1  \n",
       "\n",
       "[60182 rows x 7 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\AppData\\Local\\Temp\\ipykernel_10108\\1541996746.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df.drop(['overall', 'verified', 'reviewerID'], axis=1, inplace=True)\n"
     ]
    }
   ],
   "source": [
    "df.drop(['overall', 'verified', 'reviewerID'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>review_clean</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>It diffuses a very mild light perfume, just wh...</td>\n",
       "      <td>What I expected</td>\n",
       "      <td>wear mild perfume nice strong lotion day perfu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>One of my wife's favorites.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>one favorites wife's</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>If you have body acne this product is a must. ...</td>\n",
       "      <td>Acne Body Wash A MUST</td>\n",
       "      <td>heal help within prevent already first wash mu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Lathers well and smells great.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>well great lather smell</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>It was good, my loved her gift!</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>love good gift</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73361</th>\n",
       "      <td>Perfect</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>perfect</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73364</th>\n",
       "      <td>very good</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>good</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73365</th>\n",
       "      <td>does a great job at a great price.  Good quali...</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>great quality price good job</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73366</th>\n",
       "      <td>Microsoft's point system doesn't really make a...</td>\n",
       "      <td>The points are weird</td>\n",
       "      <td>point else sense code point good make worked g...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73367</th>\n",
       "      <td>I already had the game, but I was buying it ag...</td>\n",
       "      <td>Replacement</td>\n",
       "      <td>reason buy buy something madden great already ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>60182 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              reviewText  \\\n",
       "0      It diffuses a very mild light perfume, just wh...   \n",
       "2                            One of my wife's favorites.   \n",
       "3      If you have body acne this product is a must. ...   \n",
       "5                         Lathers well and smells great.   \n",
       "8                        It was good, my loved her gift!   \n",
       "...                                                  ...   \n",
       "73361                                            Perfect   \n",
       "73364                                          very good   \n",
       "73365  does a great job at a great price.  Good quali...   \n",
       "73366  Microsoft's point system doesn't really make a...   \n",
       "73367  I already had the game, but I was buying it ag...   \n",
       "\n",
       "                     summary  \\\n",
       "0            What I expected   \n",
       "2                 Five Stars   \n",
       "3      Acne Body Wash A MUST   \n",
       "5                 Five Stars   \n",
       "8                 Five Stars   \n",
       "...                      ...   \n",
       "73361             Five Stars   \n",
       "73364             Five Stars   \n",
       "73365             Five Stars   \n",
       "73366   The points are weird   \n",
       "73367            Replacement   \n",
       "\n",
       "                                            review_clean  sentiment  \n",
       "0      wear mild perfume nice strong lotion day perfu...          1  \n",
       "2                                   one favorites wife's          1  \n",
       "3      heal help within prevent already first wash mu...          1  \n",
       "5                                well great lather smell          1  \n",
       "8                                         love good gift          1  \n",
       "...                                                  ...        ...  \n",
       "73361                                            perfect          1  \n",
       "73364                                               good          1  \n",
       "73365                       great quality price good job          1  \n",
       "73366  point else sense code point good make worked g...          1  \n",
       "73367  reason buy buy something madden great already ...          1  \n",
       "\n",
       "[60182 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\AppData\\Local\\Temp\\ipykernel_10108\\1997736030.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df.drop('index', axis='columns', inplace=True)\n"
     ]
    }
   ],
   "source": [
    "df.reset_index(inplace=True)\n",
    "df.drop('index', axis='columns', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>review_clean</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>It diffuses a very mild light perfume, just wh...</td>\n",
       "      <td>What I expected</td>\n",
       "      <td>wear mild perfume nice strong lotion day perfu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>One of my wife's favorites.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>one favorites wife's</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>If you have body acne this product is a must. ...</td>\n",
       "      <td>Acne Body Wash A MUST</td>\n",
       "      <td>heal help within prevent already first wash mu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Lathers well and smells great.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>well great lather smell</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>It was good, my loved her gift!</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>love good gift</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60177</th>\n",
       "      <td>Perfect</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>perfect</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60178</th>\n",
       "      <td>very good</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>good</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60179</th>\n",
       "      <td>does a great job at a great price.  Good quali...</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>great quality price good job</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60180</th>\n",
       "      <td>Microsoft's point system doesn't really make a...</td>\n",
       "      <td>The points are weird</td>\n",
       "      <td>point else sense code point good make worked g...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60181</th>\n",
       "      <td>I already had the game, but I was buying it ag...</td>\n",
       "      <td>Replacement</td>\n",
       "      <td>reason buy buy something madden great already ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>60182 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              reviewText  \\\n",
       "0      It diffuses a very mild light perfume, just wh...   \n",
       "1                            One of my wife's favorites.   \n",
       "2      If you have body acne this product is a must. ...   \n",
       "3                         Lathers well and smells great.   \n",
       "4                        It was good, my loved her gift!   \n",
       "...                                                  ...   \n",
       "60177                                            Perfect   \n",
       "60178                                          very good   \n",
       "60179  does a great job at a great price.  Good quali...   \n",
       "60180  Microsoft's point system doesn't really make a...   \n",
       "60181  I already had the game, but I was buying it ag...   \n",
       "\n",
       "                     summary  \\\n",
       "0            What I expected   \n",
       "1                 Five Stars   \n",
       "2      Acne Body Wash A MUST   \n",
       "3                 Five Stars   \n",
       "4                 Five Stars   \n",
       "...                      ...   \n",
       "60177             Five Stars   \n",
       "60178             Five Stars   \n",
       "60179             Five Stars   \n",
       "60180   The points are weird   \n",
       "60181            Replacement   \n",
       "\n",
       "                                            review_clean  sentiment  \n",
       "0      wear mild perfume nice strong lotion day perfu...          1  \n",
       "1                                   one favorites wife's          1  \n",
       "2      heal help within prevent already first wash mu...          1  \n",
       "3                                well great lather smell          1  \n",
       "4                                         love good gift          1  \n",
       "...                                                  ...        ...  \n",
       "60177                                            perfect          1  \n",
       "60178                                               good          1  \n",
       "60179                       great quality price good job          1  \n",
       "60180  point else sense code point good make worked g...          1  \n",
       "60181  reason buy buy something madden great already ...          1  \n",
       "\n",
       "[60182 rows x 4 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/data_final/pre_summ.csv') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "13768953aedf00ec6bad39a037e4d4688640d7d465f42e95b481cfa8be0fcd5a"
  },
  "kernelspec": {
   "display_name": "Python 3.8.0 ('capstone3')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}