preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_json('data/org_data/Appliances_5.json', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2277 entries, 0 to 2276\n",
      "Data columns (total 12 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   overall         2277 non-null   int64 \n",
      " 1   verified        2277 non-null   bool  \n",
      " 2   reviewTime      2277 non-null   object\n",
      " 3   reviewerID      2277 non-null   object\n",
      " 4   asin            2277 non-null   object\n",
      " 5   style           38 non-null     object\n",
      " 6   reviewerName    2277 non-null   object\n",
      " 7   reviewText      2277 non-null   object\n",
      " 8   summary         2277 non-null   object\n",
      " 9   unixReviewTime  2277 non-null   int64 \n",
      " 10  vote            2074 non-null   object\n",
      " 11  image           828 non-null    object\n",
      "dtypes: bool(1), int64(2), object(9)\n",
      "memory usage: 198.0+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall</th>\n",
       "      <th>verified</th>\n",
       "      <th>reviewTime</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>style</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>unixReviewTime</th>\n",
       "      <th>vote</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>08 22, 2013</td>\n",
       "      <td>A34A1UP40713F8</td>\n",
       "      <td>B00009W3I4</td>\n",
       "      <td>{'Style:': ' Dryer Vent'}</td>\n",
       "      <td>James. Backus</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>Great product</td>\n",
       "      <td>1377129600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>02 8, 2016</td>\n",
       "      <td>A1AHW6I678O6F2</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>{'Size:': ' 6-Foot'}</td>\n",
       "      <td>kevin.</td>\n",
       "      <td>good item</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>1454889600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>08 5, 2015</td>\n",
       "      <td>A8R48NKTGCJDQ</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>{'Size:': ' 6-Foot'}</td>\n",
       "      <td>CDBrannom</td>\n",
       "      <td>Fit my new LG dryer perfectly.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>1438732800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>04 24, 2015</td>\n",
       "      <td>AR3OHHHW01A8E</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>{'Size:': ' 6-Foot'}</td>\n",
       "      <td>Calvin E Reames</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>Perfect size</td>\n",
       "      <td>1429833600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>03 21, 2015</td>\n",
       "      <td>A2CIEGHZ7L1WWR</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>{'Size:': ' 6-Foot'}</td>\n",
       "      <td>albert j. kong</td>\n",
       "      <td>Price and delivery was excellent.</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>1426896000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>12 26, 2013</td>\n",
       "      <td>A1RQQV9HYHDHDN</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>{'Size:': ' 6-Foot'}</td>\n",
       "      <td>Adam Baer</td>\n",
       "      <td>I purchasaed a new dryer and did not want to r...</td>\n",
       "      <td>Easy install</td>\n",
       "      <td>1388016000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4</td>\n",
       "      <td>True</td>\n",
       "      <td>08 2, 2016</td>\n",
       "      <td>A1OB2H416ARLFJ</td>\n",
       "      <td>B0009GVYNW</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Commando Big</td>\n",
       "      <td>Good value.</td>\n",
       "      <td>Good value for the price.</td>\n",
       "      <td>1470096000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>07 6, 2017</td>\n",
       "      <td>A3LGZ8M29PBNGG</td>\n",
       "      <td>B000N6302Q</td>\n",
       "      <td>{'Color:': ' Blue'}</td>\n",
       "      <td>nerenttt</td>\n",
       "      <td>works great. we loved ours! till we didn't.  t...</td>\n",
       "      <td>the five star posts are from new owners, the r...</td>\n",
       "      <td>1499299200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>07 6, 2017</td>\n",
       "      <td>A3LGZ8M29PBNGG</td>\n",
       "      <td>B000N6302Q</td>\n",
       "      <td>{'Color:': ' Stainless Steel'}</td>\n",
       "      <td>nerenttt</td>\n",
       "      <td>Luved it for the few months it worked!  great ...</td>\n",
       "      <td>my advice is do not buy without  extended warr...</td>\n",
       "      <td>1499299200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>11 14, 2014</td>\n",
       "      <td>A3SM7H70QI3TY8</td>\n",
       "      <td>B000N6302Q</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SRH</td>\n",
       "      <td>Be careful, NewAir will not stand by their pro...</td>\n",
       "      <td>NewAir will not stand behind their products - ...</td>\n",
       "      <td>1415923200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   overall  verified   reviewTime      reviewerID        asin  \\\n",
       "0        5      True  08 22, 2013  A34A1UP40713F8  B00009W3I4   \n",
       "1        5      True   02 8, 2016  A1AHW6I678O6F2  B00009W3PA   \n",
       "2        5      True   08 5, 2015   A8R48NKTGCJDQ  B00009W3PA   \n",
       "3        5      True  04 24, 2015   AR3OHHHW01A8E  B00009W3PA   \n",
       "4        5      True  03 21, 2015  A2CIEGHZ7L1WWR  B00009W3PA   \n",
       "5        5      True  12 26, 2013  A1RQQV9HYHDHDN  B00009W3PA   \n",
       "6        4      True   08 2, 2016  A1OB2H416ARLFJ  B0009GVYNW   \n",
       "7        2     False   07 6, 2017  A3LGZ8M29PBNGG  B000N6302Q   \n",
       "8        2     False   07 6, 2017  A3LGZ8M29PBNGG  B000N6302Q   \n",
       "9        1     False  11 14, 2014  A3SM7H70QI3TY8  B000N6302Q   \n",
       "\n",
       "                            style     reviewerName  \\\n",
       "0       {'Style:': ' Dryer Vent'}    James. Backus   \n",
       "1            {'Size:': ' 6-Foot'}           kevin.   \n",
       "2            {'Size:': ' 6-Foot'}        CDBrannom   \n",
       "3            {'Size:': ' 6-Foot'}  Calvin E Reames   \n",
       "4            {'Size:': ' 6-Foot'}   albert j. kong   \n",
       "5            {'Size:': ' 6-Foot'}        Adam Baer   \n",
       "6                             NaN     Commando Big   \n",
       "7             {'Color:': ' Blue'}         nerenttt   \n",
       "8  {'Color:': ' Stainless Steel'}         nerenttt   \n",
       "9                             NaN              SRH   \n",
       "\n",
       "                                          reviewText  \\\n",
       "0  I like this as a vent as well as something tha...   \n",
       "1                                          good item   \n",
       "2                     Fit my new LG dryer perfectly.   \n",
       "3                     Good value for electric dryers   \n",
       "4                  Price and delivery was excellent.   \n",
       "5  I purchasaed a new dryer and did not want to r...   \n",
       "6                                        Good value.   \n",
       "7  works great. we loved ours! till we didn't.  t...   \n",
       "8  Luved it for the few months it worked!  great ...   \n",
       "9  Be careful, NewAir will not stand by their pro...   \n",
       "\n",
       "                                             summary  unixReviewTime vote  \\\n",
       "0                                      Great product      1377129600  NaN   \n",
       "1                                         Five Stars      1454889600  NaN   \n",
       "2                                         Five Stars      1438732800  NaN   \n",
       "3                                       Perfect size      1429833600  NaN   \n",
       "4                                         Five Stars      1426896000  NaN   \n",
       "5                                       Easy install      1388016000  NaN   \n",
       "6                          Good value for the price.      1470096000  NaN   \n",
       "7  the five star posts are from new owners, the r...      1499299200  NaN   \n",
       "8  my advice is do not buy without  extended warr...      1499299200  NaN   \n",
       "9  NewAir will not stand behind their products - ...      1415923200  NaN   \n",
       "\n",
       "  image  \n",
       "0   NaN  \n",
       "1   NaN  \n",
       "2   NaN  \n",
       "3   NaN  \n",
       "4   NaN  \n",
       "5   NaN  \n",
       "6   NaN  \n",
       "7   NaN  \n",
       "8   NaN  \n",
       "9   NaN  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            I like this as a vent as well as something that will keep house warmer in winter.  I sanded it and then painted it the same color as the house.  Looks great.\n",
       "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                good item\n",
       "2                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           Fit my new LG dryer perfectly.\n",
       "3                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           Good value for electric dryers\n",
       "4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        Price and delivery was excellent.\n",
       "5                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             I purchasaed a new dryer and did not want to reuse the cord from my old unit. This unit installed in a pretty straight forward manor. Quality was as expected. No Complaints\n",
       "6                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              Good value.\n",
       "7                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  works great. we loved ours! till we didn't.  these do not last so buy the warranty as you WILL NEED IT.\n",
       "8                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          Luved it for the few months it worked!  great little bullet shaped ice cubes. It was a gift for my sister who never opened the box.  The next summer during a heat wave I asked for my unused gift back, ha!, and was in heaven for a few months.  the next summer after a few weeks the unit gave out.called new air and it was over the one year mark so no warranty.  so my advice is do not buy without  extended warranty!\n",
       "9    Be careful, NewAir will not stand by their products. Our NewAir broke within the first 4 hours of use.  The manufacturer will NOT cover the item under warranty. Just in case you're actually thinking of buying a NewAir product... think twice...\\n\\nFrom: NewAir Sales <sales@newair.com>\\nDate: Fri, Nov 14, 2014 at 11:03 AM\\nSubject: Re: Icemaker return\\nTo: Susan Ferguson <xxxxx@gwmail.gwu.edu>\\n\\nHi Susan,\\n\\nThe manufacturer warranty is void if the unit is purchased by a private seller who does not have our consent to sell our products. This is the same reason why we ask for a copy of your receipt upon requesting warranty claims because if it not from a valid authorized dealer we will not proceed with warranty work. We also list our authorized dealers on our web site for customer use. We stand behind our products, however when an item is purchased from a private seller the purchaser runs the risk of the item being used, open box, refurbished, and or a scratch and dent. Our one year limited manufacturer's warranty is only valid on new items sold by our authorized online dealers. It is unfortunate to see that you are in this situation, however you will need to contact the seller of the item for a replacement or refund.\\n\\nHave a great day!\\n\\nMireya\\nCustomer Support\\nNewAir | 3419 East Chapman Ave. #190 | Orange, CA 92869\\nPh: 855-963-9247 Em: sales@newair.com\\nFB: facebook.com/NewAirUSA Twitter: @NewAirUSA\\nPinterest: pinterest.com/NewAirUSA Instagram: NewAirUSA\\nWeb: [...]\\n\"Made by awesome\"\\n\\nOn Thu, Nov 13, 2014 at 7:59 PM, Susan Ferguson <xxxxx@gwmail.gwu.edu> wrote:\\nHello Mireya-\\n\\nI'm sorry but I thought the manufacturer warranty was for the manufacturer, NewAir.  Please explain why the warranty is not valid in this case.  I don't see anywhere in the warranty exclusions that the warranty is not valid if not purchased from an authorized dealer.  In addition, a consumer would never know who \"authorized dealers\" are because if a company is selling your product doesn't that implicitly make them authorized to do so?\\n\\nI sincerely hope that NewAir will reconsider this position and stand by the integrity of their products.  This ice machine was literally used for four hours before failing and it is, quite frankly egregious that your company is negating a warranty that explicitly states it will repair or replace defective materials or workmanship for up to one year.\\n\\nPlease let me know how I can talk to a dispute department regarding this matter.  I look forward to your response and I will refrain from giving any reviews of your product until this matter is resolved.\\n\\nThank you,\\nSusan Ferguson\\n\\nOn Thu, Nov 13, 2014 at 3:10 PM, NewAir Sales <sales@newair.com> wrote:\\nHi Susan,\\n\\nUnfortunately, the ice maker was not purchased from an authorized dealer. You will need to contact the seller \"Mehadrin\" through Amazon for any existing issues on your unit.\\n\\nHave a great day!\\n\\nMireya\\nCustomer Support\\nNewAir | 3419 East Chapman Ave. #190 | Orange, CA 92869\\nPh: 855-963-9247 Em: sales@newair.com\\nFB: facebook.com/NewAirUSA Twitter: @NewAirUSA\\nPinterest: pinterest.com/NewAirUSA Instagram: NewAirUSA\\nWeb: [...]\\n\"Made by awesome\"\\n\\nOn Thu, Nov 13, 2014 at 12:32 PM, Susan Ferguson <xxxxxxxxx@gwmail.gwu.edu> wrote:\\nHello-  I received an AI-100S icemaker in March 2014.  I used it one time and within 4 hours the cooling unit had shut down.  My best estimate is that the pump was still working (the water was getting pumped through) but the cooling mechanism was not.  When I plugged the unit back in to try it a few hours later, the cooling mechanism never turned on.\\n\\nSince this product is still under warranty, I am hoping to acquire a new unit since this particular one has essentially never been used and is obviously faulty in some capacity.\\n\\nI have attached the receipt for your reference.  It was purchased as a gift through Amazon Marketplace.  Thanks so much and I look forward to your response.\\n\\nSusan Ferguson\n",
       "Name: reviewText, dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['reviewText'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['reviewText'] = df['reviewText'].str.replace('\\n', ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   I like this as a vent as well as something that will keep house warmer in winter.  I sanded it and then painted it the same color as the house.  Looks great.\n",
       "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       good item\n",
       "2                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  Fit my new LG dryer perfectly.\n",
       "3                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  Good value for electric dryers\n",
       "4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               Price and delivery was excellent.\n",
       "5                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    I purchasaed a new dryer and did not want to reuse the cord from my old unit. This unit installed in a pretty straight forward manor. Quality was as expected. No Complaints\n",
       "6                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     Good value.\n",
       "7                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         works great. we loved ours! till we didn't.  these do not last so buy the warranty as you WILL NEED IT.\n",
       "8                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 Luved it for the few months it worked!  great little bullet shaped ice cubes. It was a gift for my sister who never opened the box.  The next summer during a heat wave I asked for my unused gift back, ha!, and was in heaven for a few months.  the next summer after a few weeks the unit gave out.called new air and it was over the one year mark so no warranty.  so my advice is do not buy without  extended warranty!\n",
       "9    Be careful, NewAir will not stand by their products. Our NewAir broke within the first 4 hours of use.  The manufacturer will NOT cover the item under warranty. Just in case you're actually thinking of buying a NewAir product... think twice...  From: NewAir Sales <sales@newair.com> Date: Fri, Nov 14, 2014 at 11:03 AM Subject: Re: Icemaker return To: Susan Ferguson <xxxxx@gwmail.gwu.edu>  Hi Susan,  The manufacturer warranty is void if the unit is purchased by a private seller who does not have our consent to sell our products. This is the same reason why we ask for a copy of your receipt upon requesting warranty claims because if it not from a valid authorized dealer we will not proceed with warranty work. We also list our authorized dealers on our web site for customer use. We stand behind our products, however when an item is purchased from a private seller the purchaser runs the risk of the item being used, open box, refurbished, and or a scratch and dent. Our one year limited manufacturer's warranty is only valid on new items sold by our authorized online dealers. It is unfortunate to see that you are in this situation, however you will need to contact the seller of the item for a replacement or refund.  Have a great day!  Mireya Customer Support NewAir | 3419 East Chapman Ave. #190 | Orange, CA 92869 Ph: 855-963-9247 Em: sales@newair.com FB: facebook.com/NewAirUSA Twitter: @NewAirUSA Pinterest: pinterest.com/NewAirUSA Instagram: NewAirUSA Web: [...] \"Made by awesome\"  On Thu, Nov 13, 2014 at 7:59 PM, Susan Ferguson <xxxxx@gwmail.gwu.edu> wrote: Hello Mireya-  I'm sorry but I thought the manufacturer warranty was for the manufacturer, NewAir.  Please explain why the warranty is not valid in this case.  I don't see anywhere in the warranty exclusions that the warranty is not valid if not purchased from an authorized dealer.  In addition, a consumer would never know who \"authorized dealers\" are because if a company is selling your product doesn't that implicitly make them authorized to do so?  I sincerely hope that NewAir will reconsider this position and stand by the integrity of their products.  This ice machine was literally used for four hours before failing and it is, quite frankly egregious that your company is negating a warranty that explicitly states it will repair or replace defective materials or workmanship for up to one year.  Please let me know how I can talk to a dispute department regarding this matter.  I look forward to your response and I will refrain from giving any reviews of your product until this matter is resolved.  Thank you, Susan Ferguson  On Thu, Nov 13, 2014 at 3:10 PM, NewAir Sales <sales@newair.com> wrote: Hi Susan,  Unfortunately, the ice maker was not purchased from an authorized dealer. You will need to contact the seller \"Mehadrin\" through Amazon for any existing issues on your unit.  Have a great day!  Mireya Customer Support NewAir | 3419 East Chapman Ave. #190 | Orange, CA 92869 Ph: 855-963-9247 Em: sales@newair.com FB: facebook.com/NewAirUSA Twitter: @NewAirUSA Pinterest: pinterest.com/NewAirUSA Instagram: NewAirUSA Web: [...] \"Made by awesome\"  On Thu, Nov 13, 2014 at 12:32 PM, Susan Ferguson <xxxxxxxxx@gwmail.gwu.edu> wrote: Hello-  I received an AI-100S icemaker in March 2014.  I used it one time and within 4 hours the cooling unit had shut down.  My best estimate is that the pump was still working (the water was getting pumped through) but the cooling mechanism was not.  When I plugged the unit back in to try it a few hours later, the cooling mechanism never turned on.  Since this product is still under warranty, I am hoping to acquire a new unit since this particular one has essentially never been used and is obviously faulty in some capacity.  I have attached the receipt for your reference.  It was purchased as a gift through Amazon Marketplace.  Thanks so much and I look forward to your response.  Susan Ferguson\n",
       "Name: reviewText, dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['reviewText'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', 50)   #default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                        Great product\n",
       "1                                           Five Stars\n",
       "2                                           Five Stars\n",
       "3                                         Perfect size\n",
       "4                                           Five Stars\n",
       "5                                         Easy install\n",
       "6                            Good value for the price.\n",
       "7    the five star posts are from new owners, the r...\n",
       "8    my advice is do not buy without  extended warr...\n",
       "9    NewAir will not stand behind their products - ...\n",
       "Name: summary, dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['summary'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df [['reviewerID', 'asin', 'reviewText']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A34A1UP40713F8</td>\n",
       "      <td>B00009W3I4</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1AHW6I678O6F2</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>good item</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A8R48NKTGCJDQ</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Fit my new LG dryer perfectly.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR3OHHHW01A8E</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2CIEGHZ7L1WWR</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Price and delivery was excellent.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2272</th>\n",
       "      <td>A2B2JVUX5YN8RU</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Works great. I used it and an extension kit wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2273</th>\n",
       "      <td>A1IT56MV1C09VS</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Anyone who thinks they don't have a problem wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2274</th>\n",
       "      <td>A13391AZAFJ67K</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2275</th>\n",
       "      <td>A2V0JXLJ9VCNNX</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>washer washing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2276</th>\n",
       "      <td>A3GP2HMB2AFOKF</td>\n",
       "      <td>B00W4YKCGC</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2277 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          reviewerID        asin  \\\n",
       "0     A34A1UP40713F8  B00009W3I4   \n",
       "1     A1AHW6I678O6F2  B00009W3PA   \n",
       "2      A8R48NKTGCJDQ  B00009W3PA   \n",
       "3      AR3OHHHW01A8E  B00009W3PA   \n",
       "4     A2CIEGHZ7L1WWR  B00009W3PA   \n",
       "...              ...         ...   \n",
       "2272  A2B2JVUX5YN8RU  B0006GVNOA   \n",
       "2273  A1IT56MV1C09VS  B0006GVNOA   \n",
       "2274  A13391AZAFJ67K  B00570RQ0A   \n",
       "2275  A2V0JXLJ9VCNNX  B00570RQ0A   \n",
       "2276  A3GP2HMB2AFOKF  B00W4YKCGC   \n",
       "\n",
       "                                             reviewText  \n",
       "0     I like this as a vent as well as something tha...  \n",
       "1                                             good item  \n",
       "2                        Fit my new LG dryer perfectly.  \n",
       "3                        Good value for electric dryers  \n",
       "4                     Price and delivery was excellent.  \n",
       "...                                                 ...  \n",
       "2272  Works great. I used it and an extension kit wi...  \n",
       "2273  Anyone who thinks they don't have a problem wi...  \n",
       "2274                                               good  \n",
       "2275                                     washer washing  \n",
       "2276                        Great product fast shipping  \n",
       "\n",
       "[2277 rows x 3 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# src: https://www.analyticsvidhya.com/blog/2021/06/text-preprocessing-in-nlp-with-python-codes/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Punctuation Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\anaconda3\\envs\\capstone\\lib\\site-packages\\ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>reviewText_wo_punc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A34A1UP40713F8</td>\n",
       "      <td>B00009W3I4</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1AHW6I678O6F2</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A8R48NKTGCJDQ</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Fit my new LG dryer perfectly.</td>\n",
       "      <td>Fit my new LG dryer perfectly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR3OHHHW01A8E</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2CIEGHZ7L1WWR</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Price and delivery was excellent.</td>\n",
       "      <td>Price and delivery was excellent</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       reviewerID        asin  \\\n",
       "0  A34A1UP40713F8  B00009W3I4   \n",
       "1  A1AHW6I678O6F2  B00009W3PA   \n",
       "2   A8R48NKTGCJDQ  B00009W3PA   \n",
       "3   AR3OHHHW01A8E  B00009W3PA   \n",
       "4  A2CIEGHZ7L1WWR  B00009W3PA   \n",
       "\n",
       "                                          reviewText  \\\n",
       "0  I like this as a vent as well as something tha...   \n",
       "1                                          good item   \n",
       "2                     Fit my new LG dryer perfectly.   \n",
       "3                     Good value for electric dryers   \n",
       "4                  Price and delivery was excellent.   \n",
       "\n",
       "                                  reviewText_wo_punc  \n",
       "0  I like this as a vent as well as something tha...  \n",
       "1                                          good item  \n",
       "2                      Fit my new LG dryer perfectly  \n",
       "3                     Good value for electric dryers  \n",
       "4                   Price and delivery was excellent  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#defining the function to remove punctuation\n",
    "def remove_punctuation(text):\n",
    "    punctuationfree=\"\".join([i for i in text if i not in string.punctuation])\n",
    "    return punctuationfree\n",
    "#storing the puntuation free text\n",
    "df2['reviewText_wo_punc']= df2['reviewText'].apply(lambda x:remove_punctuation(x))\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>reviewText_wo_punc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A34A1UP40713F8</td>\n",
       "      <td>B00009W3I4</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1AHW6I678O6F2</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A8R48NKTGCJDQ</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Fit my new LG dryer perfectly.</td>\n",
       "      <td>Fit my new LG dryer perfectly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR3OHHHW01A8E</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2CIEGHZ7L1WWR</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Price and delivery was excellent.</td>\n",
       "      <td>Price and delivery was excellent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2272</th>\n",
       "      <td>A2B2JVUX5YN8RU</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Works great. I used it and an extension kit wi...</td>\n",
       "      <td>Works great I used it and an extension kit wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2273</th>\n",
       "      <td>A1IT56MV1C09VS</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Anyone who thinks they don't have a problem wi...</td>\n",
       "      <td>Anyone who thinks they dont have a problem wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2274</th>\n",
       "      <td>A13391AZAFJ67K</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2275</th>\n",
       "      <td>A2V0JXLJ9VCNNX</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2276</th>\n",
       "      <td>A3GP2HMB2AFOKF</td>\n",
       "      <td>B00W4YKCGC</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2277 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          reviewerID        asin  \\\n",
       "0     A34A1UP40713F8  B00009W3I4   \n",
       "1     A1AHW6I678O6F2  B00009W3PA   \n",
       "2      A8R48NKTGCJDQ  B00009W3PA   \n",
       "3      AR3OHHHW01A8E  B00009W3PA   \n",
       "4     A2CIEGHZ7L1WWR  B00009W3PA   \n",
       "...              ...         ...   \n",
       "2272  A2B2JVUX5YN8RU  B0006GVNOA   \n",
       "2273  A1IT56MV1C09VS  B0006GVNOA   \n",
       "2274  A13391AZAFJ67K  B00570RQ0A   \n",
       "2275  A2V0JXLJ9VCNNX  B00570RQ0A   \n",
       "2276  A3GP2HMB2AFOKF  B00W4YKCGC   \n",
       "\n",
       "                                             reviewText  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                        Fit my new LG dryer perfectly.   \n",
       "3                        Good value for electric dryers   \n",
       "4                     Price and delivery was excellent.   \n",
       "...                                                 ...   \n",
       "2272  Works great. I used it and an extension kit wi...   \n",
       "2273  Anyone who thinks they don't have a problem wi...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                     reviewText_wo_punc  \n",
       "0     I like this as a vent as well as something tha...  \n",
       "1                                             good item  \n",
       "2                         Fit my new LG dryer perfectly  \n",
       "3                        Good value for electric dryers  \n",
       "4                      Price and delivery was excellent  \n",
       "...                                                 ...  \n",
       "2272  Works great I used it and an extension kit wit...  \n",
       "2273  Anyone who thinks they dont have a problem wit...  \n",
       "2274                                               good  \n",
       "2275                                     washer washing  \n",
       "2276                        Great product fast shipping  \n",
       "\n",
       "[2277 rows x 4 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lowercase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\anaconda3\\envs\\capstone\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "df2['reviewText_lower']= df2['reviewText_wo_punc'].apply(lambda x: x.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>reviewText_wo_punc</th>\n",
       "      <th>reviewText_lower</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A34A1UP40713F8</td>\n",
       "      <td>B00009W3I4</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>i like this as a vent as well as something tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1AHW6I678O6F2</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A8R48NKTGCJDQ</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Fit my new LG dryer perfectly.</td>\n",
       "      <td>Fit my new LG dryer perfectly</td>\n",
       "      <td>fit my new lg dryer perfectly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR3OHHHW01A8E</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>good value for electric dryers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2CIEGHZ7L1WWR</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Price and delivery was excellent.</td>\n",
       "      <td>Price and delivery was excellent</td>\n",
       "      <td>price and delivery was excellent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2272</th>\n",
       "      <td>A2B2JVUX5YN8RU</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Works great. I used it and an extension kit wi...</td>\n",
       "      <td>Works great I used it and an extension kit wit...</td>\n",
       "      <td>works great i used it and an extension kit wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2273</th>\n",
       "      <td>A1IT56MV1C09VS</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Anyone who thinks they don't have a problem wi...</td>\n",
       "      <td>Anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>anyone who thinks they dont have a problem wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2274</th>\n",
       "      <td>A13391AZAFJ67K</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2275</th>\n",
       "      <td>A2V0JXLJ9VCNNX</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2276</th>\n",
       "      <td>A3GP2HMB2AFOKF</td>\n",
       "      <td>B00W4YKCGC</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>great product fast shipping</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2277 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          reviewerID        asin  \\\n",
       "0     A34A1UP40713F8  B00009W3I4   \n",
       "1     A1AHW6I678O6F2  B00009W3PA   \n",
       "2      A8R48NKTGCJDQ  B00009W3PA   \n",
       "3      AR3OHHHW01A8E  B00009W3PA   \n",
       "4     A2CIEGHZ7L1WWR  B00009W3PA   \n",
       "...              ...         ...   \n",
       "2272  A2B2JVUX5YN8RU  B0006GVNOA   \n",
       "2273  A1IT56MV1C09VS  B0006GVNOA   \n",
       "2274  A13391AZAFJ67K  B00570RQ0A   \n",
       "2275  A2V0JXLJ9VCNNX  B00570RQ0A   \n",
       "2276  A3GP2HMB2AFOKF  B00W4YKCGC   \n",
       "\n",
       "                                             reviewText  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                        Fit my new LG dryer perfectly.   \n",
       "3                        Good value for electric dryers   \n",
       "4                     Price and delivery was excellent.   \n",
       "...                                                 ...   \n",
       "2272  Works great. I used it and an extension kit wi...   \n",
       "2273  Anyone who thinks they don't have a problem wi...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                     reviewText_wo_punc  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                         Fit my new LG dryer perfectly   \n",
       "3                        Good value for electric dryers   \n",
       "4                      Price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  Works great I used it and an extension kit wit...   \n",
       "2273  Anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                       reviewText_lower  \n",
       "0     i like this as a vent as well as something tha...  \n",
       "1                                             good item  \n",
       "2                         fit my new lg dryer perfectly  \n",
       "3                        good value for electric dryers  \n",
       "4                      price and delivery was excellent  \n",
       "...                                                 ...  \n",
       "2272  works great i used it and an extension kit wit...  \n",
       "2273  anyone who thinks they dont have a problem wit...  \n",
       "2274                                               good  \n",
       "2275                                     washer washing  \n",
       "2276                        great product fast shipping  \n",
       "\n",
       "[2277 rows x 5 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing Duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#src: https://stackoverflow.com/questions/47316783/python-dataframe-remove-duplicate-words-in-the-same-cell-within-a-column-in-pyt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\anaconda3\\envs\\capstone\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "df2['reviewText_no_dup'] = (df2['reviewText_lower'].str.split().apply(lambda x: OrderedDict.fromkeys(x).keys()).str.join(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>reviewText_wo_punc</th>\n",
       "      <th>reviewText_lower</th>\n",
       "      <th>reviewText_no_dup</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A34A1UP40713F8</td>\n",
       "      <td>B00009W3I4</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>i like this as a vent as well as something tha...</td>\n",
       "      <td>i like this as a vent well something that will...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1AHW6I678O6F2</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A8R48NKTGCJDQ</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Fit my new LG dryer perfectly.</td>\n",
       "      <td>Fit my new LG dryer perfectly</td>\n",
       "      <td>fit my new lg dryer perfectly</td>\n",
       "      <td>fit my new lg dryer perfectly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR3OHHHW01A8E</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>good value for electric dryers</td>\n",
       "      <td>good value for electric dryers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2CIEGHZ7L1WWR</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Price and delivery was excellent.</td>\n",
       "      <td>Price and delivery was excellent</td>\n",
       "      <td>price and delivery was excellent</td>\n",
       "      <td>price and delivery was excellent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2272</th>\n",
       "      <td>A2B2JVUX5YN8RU</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Works great. I used it and an extension kit wi...</td>\n",
       "      <td>Works great I used it and an extension kit wit...</td>\n",
       "      <td>works great i used it and an extension kit wit...</td>\n",
       "      <td>works great i used it and an extension kit wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2273</th>\n",
       "      <td>A1IT56MV1C09VS</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Anyone who thinks they don't have a problem wi...</td>\n",
       "      <td>Anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>anyone who thinks they dont have a problem wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2274</th>\n",
       "      <td>A13391AZAFJ67K</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2275</th>\n",
       "      <td>A2V0JXLJ9VCNNX</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2276</th>\n",
       "      <td>A3GP2HMB2AFOKF</td>\n",
       "      <td>B00W4YKCGC</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>great product fast shipping</td>\n",
       "      <td>great product fast shipping</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2277 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          reviewerID        asin  \\\n",
       "0     A34A1UP40713F8  B00009W3I4   \n",
       "1     A1AHW6I678O6F2  B00009W3PA   \n",
       "2      A8R48NKTGCJDQ  B00009W3PA   \n",
       "3      AR3OHHHW01A8E  B00009W3PA   \n",
       "4     A2CIEGHZ7L1WWR  B00009W3PA   \n",
       "...              ...         ...   \n",
       "2272  A2B2JVUX5YN8RU  B0006GVNOA   \n",
       "2273  A1IT56MV1C09VS  B0006GVNOA   \n",
       "2274  A13391AZAFJ67K  B00570RQ0A   \n",
       "2275  A2V0JXLJ9VCNNX  B00570RQ0A   \n",
       "2276  A3GP2HMB2AFOKF  B00W4YKCGC   \n",
       "\n",
       "                                             reviewText  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                        Fit my new LG dryer perfectly.   \n",
       "3                        Good value for electric dryers   \n",
       "4                     Price and delivery was excellent.   \n",
       "...                                                 ...   \n",
       "2272  Works great. I used it and an extension kit wi...   \n",
       "2273  Anyone who thinks they don't have a problem wi...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                     reviewText_wo_punc  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                         Fit my new LG dryer perfectly   \n",
       "3                        Good value for electric dryers   \n",
       "4                      Price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  Works great I used it and an extension kit wit...   \n",
       "2273  Anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                       reviewText_lower  \\\n",
       "0     i like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                         fit my new lg dryer perfectly   \n",
       "3                        good value for electric dryers   \n",
       "4                      price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  works great i used it and an extension kit wit...   \n",
       "2273  anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        great product fast shipping   \n",
       "\n",
       "                                      reviewText_no_dup  \n",
       "0     i like this as a vent well something that will...  \n",
       "1                                             good item  \n",
       "2                         fit my new lg dryer perfectly  \n",
       "3                        good value for electric dryers  \n",
       "4                      price and delivery was excellent  \n",
       "...                                                 ...  \n",
       "2272  works great i used it and an extension kit wit...  \n",
       "2273  anyone who thinks they dont have a problem wit...  \n",
       "2274                                               good  \n",
       "2275                                     washer washing  \n",
       "2276                        great product fast shipping  \n",
       "\n",
       "[2277 rows x 6 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Admin\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers\\punkt.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenization(text):\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "    return tokens\n",
    "#applying function to the column\n",
    "df2['reviewText_token']= df2['reviewText_no_dup'].apply(lambda x: tokenization(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>reviewText_wo_punc</th>\n",
       "      <th>reviewText_lower</th>\n",
       "      <th>reviewText_no_dup</th>\n",
       "      <th>reviewText_token</th>\n",
       "      <th>reviewText_no_stopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A34A1UP40713F8</td>\n",
       "      <td>B00009W3I4</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>i like this as a vent as well as something tha...</td>\n",
       "      <td>i like this as a vent well something that will...</td>\n",
       "      <td>[i, like, this, as, a, vent, well, something, ...</td>\n",
       "      <td>[i like this as a vent well something that wil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1AHW6I678O6F2</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>[good, item]</td>\n",
       "      <td>[good item]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A8R48NKTGCJDQ</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Fit my new LG dryer perfectly.</td>\n",
       "      <td>Fit my new LG dryer perfectly</td>\n",
       "      <td>fit my new lg dryer perfectly</td>\n",
       "      <td>fit my new lg dryer perfectly</td>\n",
       "      <td>[fit, my, new, lg, dryer, perfectly]</td>\n",
       "      <td>[fit my new lg dryer perfectly]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR3OHHHW01A8E</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>good value for electric dryers</td>\n",
       "      <td>good value for electric dryers</td>\n",
       "      <td>[good, value, for, electric, dryers]</td>\n",
       "      <td>[good value for electric dryers]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2CIEGHZ7L1WWR</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Price and delivery was excellent.</td>\n",
       "      <td>Price and delivery was excellent</td>\n",
       "      <td>price and delivery was excellent</td>\n",
       "      <td>price and delivery was excellent</td>\n",
       "      <td>[price, and, delivery, was, excellent]</td>\n",
       "      <td>[price and delivery was excellent]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2272</th>\n",
       "      <td>A2B2JVUX5YN8RU</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Works great. I used it and an extension kit wi...</td>\n",
       "      <td>Works great I used it and an extension kit wit...</td>\n",
       "      <td>works great i used it and an extension kit wit...</td>\n",
       "      <td>works great i used it and an extension kit wit...</td>\n",
       "      <td>[works, great, i, used, it, and, an, extension...</td>\n",
       "      <td>[works great i used it and an extension kit wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2273</th>\n",
       "      <td>A1IT56MV1C09VS</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Anyone who thinks they don't have a problem wi...</td>\n",
       "      <td>Anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>[anyone, who, thinks, they, dont, have, a, pro...</td>\n",
       "      <td>[anyone who thinks they dont have a problem wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2274</th>\n",
       "      <td>A13391AZAFJ67K</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>[good]</td>\n",
       "      <td>[good]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2275</th>\n",
       "      <td>A2V0JXLJ9VCNNX</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>[washer, washing]</td>\n",
       "      <td>[washer washing]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2276</th>\n",
       "      <td>A3GP2HMB2AFOKF</td>\n",
       "      <td>B00W4YKCGC</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>great product fast shipping</td>\n",
       "      <td>great product fast shipping</td>\n",
       "      <td>[great, product, fast, shipping]</td>\n",
       "      <td>[great product fast shipping]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2277 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          reviewerID        asin  \\\n",
       "0     A34A1UP40713F8  B00009W3I4   \n",
       "1     A1AHW6I678O6F2  B00009W3PA   \n",
       "2      A8R48NKTGCJDQ  B00009W3PA   \n",
       "3      AR3OHHHW01A8E  B00009W3PA   \n",
       "4     A2CIEGHZ7L1WWR  B00009W3PA   \n",
       "...              ...         ...   \n",
       "2272  A2B2JVUX5YN8RU  B0006GVNOA   \n",
       "2273  A1IT56MV1C09VS  B0006GVNOA   \n",
       "2274  A13391AZAFJ67K  B00570RQ0A   \n",
       "2275  A2V0JXLJ9VCNNX  B00570RQ0A   \n",
       "2276  A3GP2HMB2AFOKF  B00W4YKCGC   \n",
       "\n",
       "                                             reviewText  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                        Fit my new LG dryer perfectly.   \n",
       "3                        Good value for electric dryers   \n",
       "4                     Price and delivery was excellent.   \n",
       "...                                                 ...   \n",
       "2272  Works great. I used it and an extension kit wi...   \n",
       "2273  Anyone who thinks they don't have a problem wi...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                     reviewText_wo_punc  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                         Fit my new LG dryer perfectly   \n",
       "3                        Good value for electric dryers   \n",
       "4                      Price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  Works great I used it and an extension kit wit...   \n",
       "2273  Anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                       reviewText_lower  \\\n",
       "0     i like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                         fit my new lg dryer perfectly   \n",
       "3                        good value for electric dryers   \n",
       "4                      price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  works great i used it and an extension kit wit...   \n",
       "2273  anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        great product fast shipping   \n",
       "\n",
       "                                      reviewText_no_dup  \\\n",
       "0     i like this as a vent well something that will...   \n",
       "1                                             good item   \n",
       "2                         fit my new lg dryer perfectly   \n",
       "3                        good value for electric dryers   \n",
       "4                      price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  works great i used it and an extension kit wit...   \n",
       "2273  anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        great product fast shipping   \n",
       "\n",
       "                                       reviewText_token  \\\n",
       "0     [i, like, this, as, a, vent, well, something, ...   \n",
       "1                                          [good, item]   \n",
       "2                  [fit, my, new, lg, dryer, perfectly]   \n",
       "3                  [good, value, for, electric, dryers]   \n",
       "4                [price, and, delivery, was, excellent]   \n",
       "...                                                 ...   \n",
       "2272  [works, great, i, used, it, and, an, extension...   \n",
       "2273  [anyone, who, thinks, they, dont, have, a, pro...   \n",
       "2274                                             [good]   \n",
       "2275                                  [washer, washing]   \n",
       "2276                   [great, product, fast, shipping]   \n",
       "\n",
       "                                reviewText_no_stopwords  \n",
       "0     [i like this as a vent well something that wil...  \n",
       "1                                           [good item]  \n",
       "2                       [fit my new lg dryer perfectly]  \n",
       "3                      [good value for electric dryers]  \n",
       "4                    [price and delivery was excellent]  \n",
       "...                                                 ...  \n",
       "2272  [works great i used it and an extension kit wi...  \n",
       "2273  [anyone who thinks they dont have a problem wi...  \n",
       "2274                                             [good]  \n",
       "2275                                   [washer washing]  \n",
       "2276                      [great product fast shipping]  \n",
       "\n",
       "[2277 rows x 8 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Stop Words Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Admin\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'me',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'we',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'you',\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " \"you'll\",\n",
       " \"you'd\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves',\n",
       " 'he',\n",
       " 'him',\n",
       " 'his',\n",
       " 'himself',\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'her',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'they',\n",
       " 'them',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'themselves',\n",
       " 'what',\n",
       " 'which',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'this',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'these',\n",
       " 'those',\n",
       " 'am',\n",
       " 'is',\n",
       " 'are',\n",
       " 'was',\n",
       " 'were',\n",
       " 'be',\n",
       " 'been',\n",
       " 'being',\n",
       " 'have',\n",
       " 'has',\n",
       " 'had',\n",
       " 'having',\n",
       " 'do',\n",
       " 'does',\n",
       " 'did',\n",
       " 'doing',\n",
       " 'a',\n",
       " 'an',\n",
       " 'the',\n",
       " 'and',\n",
       " 'but',\n",
       " 'if',\n",
       " 'or',\n",
       " 'because',\n",
       " 'as',\n",
       " 'until',\n",
       " 'while',\n",
       " 'of',\n",
       " 'at',\n",
       " 'by',\n",
       " 'for',\n",
       " 'with',\n",
       " 'about',\n",
       " 'against',\n",
       " 'between',\n",
       " 'into',\n",
       " 'through',\n",
       " 'during',\n",
       " 'before',\n",
       " 'after',\n",
       " 'above',\n",
       " 'below',\n",
       " 'to',\n",
       " 'from',\n",
       " 'up',\n",
       " 'down',\n",
       " 'in',\n",
       " 'out',\n",
       " 'on',\n",
       " 'off',\n",
       " 'over',\n",
       " 'under',\n",
       " 'again',\n",
       " 'further',\n",
       " 'then',\n",
       " 'once',\n",
       " 'here',\n",
       " 'there',\n",
       " 'when',\n",
       " 'where',\n",
       " 'why',\n",
       " 'how',\n",
       " 'all',\n",
       " 'any',\n",
       " 'both',\n",
       " 'each',\n",
       " 'few',\n",
       " 'more',\n",
       " 'most',\n",
       " 'other',\n",
       " 'some',\n",
       " 'such',\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'only',\n",
       " 'own',\n",
       " 'same',\n",
       " 'so',\n",
       " 'than',\n",
       " 'too',\n",
       " 'very',\n",
       " 's',\n",
       " 't',\n",
       " 'can',\n",
       " 'will',\n",
       " 'just',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'now',\n",
       " 'd',\n",
       " 'll',\n",
       " 'm',\n",
       " 'o',\n",
       " 're',\n",
       " 've',\n",
       " 'y',\n",
       " 'ain',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'ma',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\"]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords = nltk.corpus.stopwords.words('english')\n",
    "stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(text):\n",
    "    output= [i for i in text if i not in stopwords]\n",
    "    return output\n",
    "\n",
    "df2['reviewText_no_stopwords']= df2['reviewText_token'].apply(lambda x:remove_stopwords(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>reviewText_wo_punc</th>\n",
       "      <th>reviewText_lower</th>\n",
       "      <th>reviewText_no_dup</th>\n",
       "      <th>reviewText_token</th>\n",
       "      <th>reviewText_no_stopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A34A1UP40713F8</td>\n",
       "      <td>B00009W3I4</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>i like this as a vent as well as something tha...</td>\n",
       "      <td>i like this as a vent well something that will...</td>\n",
       "      <td>[i, like, this, as, a, vent, well, something, ...</td>\n",
       "      <td>[like, vent, well, something, keep, house, war...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1AHW6I678O6F2</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>[good, item]</td>\n",
       "      <td>[good, item]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A8R48NKTGCJDQ</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Fit my new LG dryer perfectly.</td>\n",
       "      <td>Fit my new LG dryer perfectly</td>\n",
       "      <td>fit my new lg dryer perfectly</td>\n",
       "      <td>fit my new lg dryer perfectly</td>\n",
       "      <td>[fit, my, new, lg, dryer, perfectly]</td>\n",
       "      <td>[fit, new, lg, dryer, perfectly]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR3OHHHW01A8E</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>good value for electric dryers</td>\n",
       "      <td>good value for electric dryers</td>\n",
       "      <td>[good, value, for, electric, dryers]</td>\n",
       "      <td>[good, value, electric, dryers]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2CIEGHZ7L1WWR</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Price and delivery was excellent.</td>\n",
       "      <td>Price and delivery was excellent</td>\n",
       "      <td>price and delivery was excellent</td>\n",
       "      <td>price and delivery was excellent</td>\n",
       "      <td>[price, and, delivery, was, excellent]</td>\n",
       "      <td>[price, delivery, excellent]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2272</th>\n",
       "      <td>A2B2JVUX5YN8RU</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Works great. I used it and an extension kit wi...</td>\n",
       "      <td>Works great I used it and an extension kit wit...</td>\n",
       "      <td>works great i used it and an extension kit wit...</td>\n",
       "      <td>works great i used it and an extension kit wit...</td>\n",
       "      <td>[works, great, i, used, it, and, an, extension...</td>\n",
       "      <td>[works, great, used, extension, kit, extra, ro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2273</th>\n",
       "      <td>A1IT56MV1C09VS</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Anyone who thinks they don't have a problem wi...</td>\n",
       "      <td>Anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>[anyone, who, thinks, they, dont, have, a, pro...</td>\n",
       "      <td>[anyone, thinks, dont, problem, lint, look, pi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2274</th>\n",
       "      <td>A13391AZAFJ67K</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>[good]</td>\n",
       "      <td>[good]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2275</th>\n",
       "      <td>A2V0JXLJ9VCNNX</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>[washer, washing]</td>\n",
       "      <td>[washer, washing]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2276</th>\n",
       "      <td>A3GP2HMB2AFOKF</td>\n",
       "      <td>B00W4YKCGC</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>great product fast shipping</td>\n",
       "      <td>great product fast shipping</td>\n",
       "      <td>[great, product, fast, shipping]</td>\n",
       "      <td>[great, product, fast, shipping]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2277 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          reviewerID        asin  \\\n",
       "0     A34A1UP40713F8  B00009W3I4   \n",
       "1     A1AHW6I678O6F2  B00009W3PA   \n",
       "2      A8R48NKTGCJDQ  B00009W3PA   \n",
       "3      AR3OHHHW01A8E  B00009W3PA   \n",
       "4     A2CIEGHZ7L1WWR  B00009W3PA   \n",
       "...              ...         ...   \n",
       "2272  A2B2JVUX5YN8RU  B0006GVNOA   \n",
       "2273  A1IT56MV1C09VS  B0006GVNOA   \n",
       "2274  A13391AZAFJ67K  B00570RQ0A   \n",
       "2275  A2V0JXLJ9VCNNX  B00570RQ0A   \n",
       "2276  A3GP2HMB2AFOKF  B00W4YKCGC   \n",
       "\n",
       "                                             reviewText  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                        Fit my new LG dryer perfectly.   \n",
       "3                        Good value for electric dryers   \n",
       "4                     Price and delivery was excellent.   \n",
       "...                                                 ...   \n",
       "2272  Works great. I used it and an extension kit wi...   \n",
       "2273  Anyone who thinks they don't have a problem wi...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                     reviewText_wo_punc  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                         Fit my new LG dryer perfectly   \n",
       "3                        Good value for electric dryers   \n",
       "4                      Price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  Works great I used it and an extension kit wit...   \n",
       "2273  Anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                       reviewText_lower  \\\n",
       "0     i like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                         fit my new lg dryer perfectly   \n",
       "3                        good value for electric dryers   \n",
       "4                      price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  works great i used it and an extension kit wit...   \n",
       "2273  anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        great product fast shipping   \n",
       "\n",
       "                                      reviewText_no_dup  \\\n",
       "0     i like this as a vent well something that will...   \n",
       "1                                             good item   \n",
       "2                         fit my new lg dryer perfectly   \n",
       "3                        good value for electric dryers   \n",
       "4                      price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  works great i used it and an extension kit wit...   \n",
       "2273  anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        great product fast shipping   \n",
       "\n",
       "                                       reviewText_token  \\\n",
       "0     [i, like, this, as, a, vent, well, something, ...   \n",
       "1                                          [good, item]   \n",
       "2                  [fit, my, new, lg, dryer, perfectly]   \n",
       "3                  [good, value, for, electric, dryers]   \n",
       "4                [price, and, delivery, was, excellent]   \n",
       "...                                                 ...   \n",
       "2272  [works, great, i, used, it, and, an, extension...   \n",
       "2273  [anyone, who, thinks, they, dont, have, a, pro...   \n",
       "2274                                             [good]   \n",
       "2275                                  [washer, washing]   \n",
       "2276                   [great, product, fast, shipping]   \n",
       "\n",
       "                                reviewText_no_stopwords  \n",
       "0     [like, vent, well, something, keep, house, war...  \n",
       "1                                          [good, item]  \n",
       "2                      [fit, new, lg, dryer, perfectly]  \n",
       "3                       [good, value, electric, dryers]  \n",
       "4                          [price, delivery, excellent]  \n",
       "...                                                 ...  \n",
       "2272  [works, great, used, extension, kit, extra, ro...  \n",
       "2273  [anyone, thinks, dont, problem, lint, look, pi...  \n",
       "2274                                             [good]  \n",
       "2275                                  [washer, washing]  \n",
       "2276                   [great, product, fast, shipping]  \n",
       "\n",
       "[2277 rows x 8 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\Admin\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     C:\\Users\\Admin\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping corpora\\omw-1.4.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('wordnet')\n",
    "nltk.download('omw-1.4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordnet_lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatizer(text):\n",
    "    lemm_text = [wordnet_lemmatizer.lemmatize(word) for word in text]\n",
    "    return lemm_text\n",
    "df2['reviewText_lemm']=df2['reviewText_no_stopwords'].apply(lambda x:lemmatizer(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>reviewText_wo_punc</th>\n",
       "      <th>reviewText_lower</th>\n",
       "      <th>reviewText_no_dup</th>\n",
       "      <th>reviewText_token</th>\n",
       "      <th>reviewText_no_stopwords</th>\n",
       "      <th>reviewText_lemm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A34A1UP40713F8</td>\n",
       "      <td>B00009W3I4</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>I like this as a vent as well as something tha...</td>\n",
       "      <td>i like this as a vent as well as something tha...</td>\n",
       "      <td>i like this as a vent well something that will...</td>\n",
       "      <td>[i, like, this, as, a, vent, well, something, ...</td>\n",
       "      <td>[like, vent, well, something, keep, house, war...</td>\n",
       "      <td>[like, vent, well, something, keep, house, war...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1AHW6I678O6F2</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>good item</td>\n",
       "      <td>[good, item]</td>\n",
       "      <td>[good, item]</td>\n",
       "      <td>[good, item]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A8R48NKTGCJDQ</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Fit my new LG dryer perfectly.</td>\n",
       "      <td>Fit my new LG dryer perfectly</td>\n",
       "      <td>fit my new lg dryer perfectly</td>\n",
       "      <td>fit my new lg dryer perfectly</td>\n",
       "      <td>[fit, my, new, lg, dryer, perfectly]</td>\n",
       "      <td>[fit, new, lg, dryer, perfectly]</td>\n",
       "      <td>[fit, new, lg, dryer, perfectly]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR3OHHHW01A8E</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>Good value for electric dryers</td>\n",
       "      <td>good value for electric dryers</td>\n",
       "      <td>good value for electric dryers</td>\n",
       "      <td>[good, value, for, electric, dryers]</td>\n",
       "      <td>[good, value, electric, dryers]</td>\n",
       "      <td>[good, value, electric, dryer]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2CIEGHZ7L1WWR</td>\n",
       "      <td>B00009W3PA</td>\n",
       "      <td>Price and delivery was excellent.</td>\n",
       "      <td>Price and delivery was excellent</td>\n",
       "      <td>price and delivery was excellent</td>\n",
       "      <td>price and delivery was excellent</td>\n",
       "      <td>[price, and, delivery, was, excellent]</td>\n",
       "      <td>[price, delivery, excellent]</td>\n",
       "      <td>[price, delivery, excellent]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2272</th>\n",
       "      <td>A2B2JVUX5YN8RU</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Works great. I used it and an extension kit wi...</td>\n",
       "      <td>Works great I used it and an extension kit wit...</td>\n",
       "      <td>works great i used it and an extension kit wit...</td>\n",
       "      <td>works great i used it and an extension kit wit...</td>\n",
       "      <td>[works, great, i, used, it, and, an, extension...</td>\n",
       "      <td>[works, great, used, extension, kit, extra, ro...</td>\n",
       "      <td>[work, great, used, extension, kit, extra, rod...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2273</th>\n",
       "      <td>A1IT56MV1C09VS</td>\n",
       "      <td>B0006GVNOA</td>\n",
       "      <td>Anyone who thinks they don't have a problem wi...</td>\n",
       "      <td>Anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>anyone who thinks they dont have a problem wit...</td>\n",
       "      <td>[anyone, who, thinks, they, dont, have, a, pro...</td>\n",
       "      <td>[anyone, thinks, dont, problem, lint, look, pi...</td>\n",
       "      <td>[anyone, think, dont, problem, lint, look, pic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2274</th>\n",
       "      <td>A13391AZAFJ67K</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>good</td>\n",
       "      <td>[good]</td>\n",
       "      <td>[good]</td>\n",
       "      <td>[good]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2275</th>\n",
       "      <td>A2V0JXLJ9VCNNX</td>\n",
       "      <td>B00570RQ0A</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>washer washing</td>\n",
       "      <td>[washer, washing]</td>\n",
       "      <td>[washer, washing]</td>\n",
       "      <td>[washer, washing]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2276</th>\n",
       "      <td>A3GP2HMB2AFOKF</td>\n",
       "      <td>B00W4YKCGC</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>Great product fast shipping</td>\n",
       "      <td>great product fast shipping</td>\n",
       "      <td>great product fast shipping</td>\n",
       "      <td>[great, product, fast, shipping]</td>\n",
       "      <td>[great, product, fast, shipping]</td>\n",
       "      <td>[great, product, fast, shipping]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2277 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          reviewerID        asin  \\\n",
       "0     A34A1UP40713F8  B00009W3I4   \n",
       "1     A1AHW6I678O6F2  B00009W3PA   \n",
       "2      A8R48NKTGCJDQ  B00009W3PA   \n",
       "3      AR3OHHHW01A8E  B00009W3PA   \n",
       "4     A2CIEGHZ7L1WWR  B00009W3PA   \n",
       "...              ...         ...   \n",
       "2272  A2B2JVUX5YN8RU  B0006GVNOA   \n",
       "2273  A1IT56MV1C09VS  B0006GVNOA   \n",
       "2274  A13391AZAFJ67K  B00570RQ0A   \n",
       "2275  A2V0JXLJ9VCNNX  B00570RQ0A   \n",
       "2276  A3GP2HMB2AFOKF  B00W4YKCGC   \n",
       "\n",
       "                                             reviewText  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                        Fit my new LG dryer perfectly.   \n",
       "3                        Good value for electric dryers   \n",
       "4                     Price and delivery was excellent.   \n",
       "...                                                 ...   \n",
       "2272  Works great. I used it and an extension kit wi...   \n",
       "2273  Anyone who thinks they don't have a problem wi...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                     reviewText_wo_punc  \\\n",
       "0     I like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                         Fit my new LG dryer perfectly   \n",
       "3                        Good value for electric dryers   \n",
       "4                      Price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  Works great I used it and an extension kit wit...   \n",
       "2273  Anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        Great product fast shipping   \n",
       "\n",
       "                                       reviewText_lower  \\\n",
       "0     i like this as a vent as well as something tha...   \n",
       "1                                             good item   \n",
       "2                         fit my new lg dryer perfectly   \n",
       "3                        good value for electric dryers   \n",
       "4                      price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  works great i used it and an extension kit wit...   \n",
       "2273  anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        great product fast shipping   \n",
       "\n",
       "                                      reviewText_no_dup  \\\n",
       "0     i like this as a vent well something that will...   \n",
       "1                                             good item   \n",
       "2                         fit my new lg dryer perfectly   \n",
       "3                        good value for electric dryers   \n",
       "4                      price and delivery was excellent   \n",
       "...                                                 ...   \n",
       "2272  works great i used it and an extension kit wit...   \n",
       "2273  anyone who thinks they dont have a problem wit...   \n",
       "2274                                               good   \n",
       "2275                                     washer washing   \n",
       "2276                        great product fast shipping   \n",
       "\n",
       "                                       reviewText_token  \\\n",
       "0     [i, like, this, as, a, vent, well, something, ...   \n",
       "1                                          [good, item]   \n",
       "2                  [fit, my, new, lg, dryer, perfectly]   \n",
       "3                  [good, value, for, electric, dryers]   \n",
       "4                [price, and, delivery, was, excellent]   \n",
       "...                                                 ...   \n",
       "2272  [works, great, i, used, it, and, an, extension...   \n",
       "2273  [anyone, who, thinks, they, dont, have, a, pro...   \n",
       "2274                                             [good]   \n",
       "2275                                  [washer, washing]   \n",
       "2276                   [great, product, fast, shipping]   \n",
       "\n",
       "                                reviewText_no_stopwords  \\\n",
       "0     [like, vent, well, something, keep, house, war...   \n",
       "1                                          [good, item]   \n",
       "2                      [fit, new, lg, dryer, perfectly]   \n",
       "3                       [good, value, electric, dryers]   \n",
       "4                          [price, delivery, excellent]   \n",
       "...                                                 ...   \n",
       "2272  [works, great, used, extension, kit, extra, ro...   \n",
       "2273  [anyone, thinks, dont, problem, lint, look, pi...   \n",
       "2274                                             [good]   \n",
       "2275                                  [washer, washing]   \n",
       "2276                   [great, product, fast, shipping]   \n",
       "\n",
       "                                        reviewText_lemm  \n",
       "0     [like, vent, well, something, keep, house, war...  \n",
       "1                                          [good, item]  \n",
       "2                      [fit, new, lg, dryer, perfectly]  \n",
       "3                        [good, value, electric, dryer]  \n",
       "4                          [price, delivery, excellent]  \n",
       "...                                                 ...  \n",
       "2272  [work, great, used, extension, kit, extra, rod...  \n",
       "2273  [anyone, think, dont, problem, lint, look, pic...  \n",
       "2274                                             [good]  \n",
       "2275                                  [washer, washing]  \n",
       "2276                   [great, product, fast, shipping]  \n",
       "\n",
       "[2277 rows x 9 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TF-IDF Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorized_text = vectorizer.fit_transform(df2['reviewText_no_dup'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<2277x1426 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 343847 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorized_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorized_text.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.01087584, 0.01089978, 0.01097195, 0.01100097,\n",
       "       0.01103492, 0.0110495 , 0.01105437, 0.01109338, 0.01114236,\n",
       "       0.01213671, 0.01214212, 0.01214754, 0.01215837, 0.01218009,\n",
       "       0.01221819, 0.01332898, 0.013439  , 0.01471348, 0.01474834,\n",
       "       0.01475533, 0.01476232, 0.01476932, 0.01479735, 0.01480437,\n",
       "       0.01481842, 0.0148325 , 0.01485365, 0.01487484, 0.01622594,\n",
       "       0.01625827, 0.0162907 , 0.01629882, 0.01632323, 0.01633138,\n",
       "       0.01755385, 0.0175925 , 0.01770899, 0.01771745, 0.01775581,\n",
       "       0.01781061, 0.01784201, 0.01790498, 0.01798403, 0.01807849,\n",
       "       0.01817566, 0.01821479, 0.01822459, 0.01826391, 0.01827376,\n",
       "       0.01828362, 0.01958893, 0.01959767, 0.0196239 , 0.01965894,\n",
       "       0.0197557 , 0.01980871, 0.02046367, 0.02050036, 0.02051263,\n",
       "       0.0205249 , 0.02053719, 0.02056182, 0.02057415, 0.02151329,\n",
       "       0.02169086, 0.02176044, 0.0228445 , 0.02289479, 0.02290752,\n",
       "       0.02295796, 0.02304639, 0.02310733, 0.02310997, 0.02317108,\n",
       "       0.02317864, 0.02320564, 0.0232195 , 0.02324259, 0.02326969,\n",
       "       0.02327331, 0.02328356, 0.02330145, 0.02330186, 0.02335031,\n",
       "       0.02336574, 0.02336652, 0.02341528, 0.02343158, 0.02344791,\n",
       "       0.02346427, 0.0234689 , 0.02349707, 0.0235135 , 0.02352996,\n",
       "       0.02380417, 0.02381545, 0.02382673, 0.02391728, 0.02397564,\n",
       "       0.02402842, 0.02418753, 0.02425148, 0.02432633, 0.02435848,\n",
       "       0.02436921, 0.02445522, 0.02456319, 0.02549294, 0.02550431,\n",
       "       0.02551568, 0.02553845, 0.02556327, 0.02557467, 0.02558405,\n",
       "       0.02558608, 0.02560891, 0.02565464, 0.02570997, 0.0257349 ,\n",
       "       0.02577896, 0.0257809 , 0.02585008, 0.02588287, 0.02594267,\n",
       "       0.02595428, 0.02624123, 0.02630668, 0.02634608, 0.02635923,\n",
       "       0.02641195, 0.02641647, 0.02643837, 0.02647463, 0.0265578 ,\n",
       "       0.02657044, 0.02662894, 0.02664993, 0.02666555, 0.0267204 ,\n",
       "       0.02672426, 0.02675522, 0.02680286, 0.02680298, 0.02680526,\n",
       "       0.02683829, 0.02685011, 0.02687614, 0.02693485, 0.02694488,\n",
       "       0.02695909, 0.02697235, 0.02699472, 0.02700661, 0.02706384,\n",
       "       0.02709135, 0.02719894, 0.02722159, 0.02731439, 0.0274585 ,\n",
       "       0.02748272, 0.02750699, 0.02752326, 0.02753132, 0.02755571,\n",
       "       0.02758016, 0.02758386, 0.02760467, 0.02765386, 0.02776651,\n",
       "       0.02783993, 0.02792585, 0.02796276, 0.02797507, 0.02799729,\n",
       "       0.02807381, 0.02807453, 0.02819776, 0.02839705, 0.02895215,\n",
       "       0.02901589, 0.02917911, 0.02920802, 0.02928525, 0.0293517 ,\n",
       "       0.02937564, 0.02939909, 0.02942742, 0.02943076, 0.02947837,\n",
       "       0.02947902, 0.02949217, 0.02949427, 0.02950532, 0.02954207,\n",
       "       0.02957401, 0.02958438, 0.02965085, 0.02966407, 0.0296773 ,\n",
       "       0.02970378, 0.02975682, 0.02977025, 0.02978353, 0.0298101 ,\n",
       "       0.0305523 , 0.03061957, 0.03071413, 0.03072783, 0.03074153,\n",
       "       0.03076896, 0.03082232, 0.03082391, 0.03090382, 0.03092034,\n",
       "       0.03096933, 0.03097419, 0.03099067, 0.03099331, 0.0309992 ,\n",
       "       0.03102269, 0.03103751, 0.03104017, 0.03104238, 0.03105384,\n",
       "       0.03105873, 0.0310641 , 0.03107881, 0.03108157, 0.03109354,\n",
       "       0.03109631, 0.03110828, 0.03116732, 0.03116998, 0.03118211,\n",
       "       0.03118392, 0.03121171, 0.03124135, 0.03124303, 0.03124793,\n",
       "       0.03125183, 0.03130103, 0.03132064, 0.03132564, 0.03133053,\n",
       "       0.03133056, 0.03133397, 0.03136033, 0.03136403, 0.03140296,\n",
       "       0.03142232, 0.03142725, 0.03143308, 0.03146385, 0.03147771,\n",
       "       0.03148265, 0.03152803, 0.03159376, 0.03161089, 0.03164122,\n",
       "       0.03169448, 0.03170896, 0.03172488, 0.03173544, 0.03175087,\n",
       "       0.03176485, 0.0317923 , 0.03180531, 0.03182279, 0.03183432,\n",
       "       0.03184834, 0.03186069, 0.03186485, 0.03187696, 0.03187889,\n",
       "       0.03196074, 0.0319914 , 0.03201591, 0.0320177 , 0.03210185,\n",
       "       0.03213265, 0.03214227, 0.03219964, 0.03222726, 0.0322422 ,\n",
       "       0.0322564 , 0.03230867, 0.03232672, 0.03237024, 0.03237495,\n",
       "       0.0323837 , 0.03242415, 0.03251253, 0.03251316, 0.03252793,\n",
       "       0.03254335, 0.03255877, 0.03266702, 0.03267117, 0.03269805,\n",
       "       0.03274688, 0.03289147, 0.03289946, 0.03294996, 0.03302884,\n",
       "       0.03310786, 0.0331173 , 0.03312767, 0.03324705, 0.03335137,\n",
       "       0.03336606, 0.03348382, 0.03373139, 0.03393854, 0.03397536,\n",
       "       0.03400982, 0.03401326, 0.03405016, 0.03409434, 0.03410099,\n",
       "       0.03412476, 0.03414739, 0.0341502 , 0.0341552 , 0.03417633,\n",
       "       0.03421832, 0.03423539, 0.03423848, 0.03424442, 0.03427563,\n",
       "       0.03428666, 0.03428787, 0.03430378, 0.03431273, 0.03432325,\n",
       "       0.03432902, 0.03432984, 0.03433513, 0.03436626, 0.03438125,\n",
       "       0.03438261, 0.03440677, 0.03443031, 0.03443496, 0.03446721,\n",
       "       0.03447232, 0.03447823, 0.03448048, 0.03449566, 0.03451789,\n",
       "       0.03452638, 0.03453309, 0.03454487, 0.03455971, 0.03456514,\n",
       "       0.03457476, 0.03457953, 0.03459054, 0.03459597, 0.03461741,\n",
       "       0.03462141, 0.03462337, 0.03462684, 0.03464017, 0.03465497,\n",
       "       0.03465755, 0.03467221, 0.03467493, 0.03468868, 0.03469233,\n",
       "       0.03477025, 0.0347972 , 0.03480797, 0.0348594 , 0.03487497,\n",
       "       0.03493722, 0.03496663, 0.03498222, 0.03499782, 0.03500017,\n",
       "       0.03502905, 0.03503139, 0.03506265, 0.03510911, 0.03512526,\n",
       "       0.03520139, 0.03523516, 0.03526431, 0.03529814, 0.03535894,\n",
       "       0.03539286, 0.03541464, 0.03547786, 0.03548258, 0.03550146,\n",
       "       0.03551963, 0.03553552, 0.03561788, 0.03565241, 0.03579312,\n",
       "       0.03585641, 0.0358734 , 0.03591264, 0.03593054, 0.03609168,\n",
       "       0.03611041, 0.03615098, 0.03616024, 0.03617114, 0.03623685,\n",
       "       0.03636657, 0.03641065, 0.03650693, 0.03666564, 0.036668  ,\n",
       "       0.03668415, 0.03676388, 0.03695759, 0.03701515, 0.03721518,\n",
       "       0.03723512, 0.03728421, 0.03732335, 0.03734103, 0.03734228,\n",
       "       0.03735872, 0.03737642, 0.03744736, 0.03746513, 0.0375007 ,\n",
       "       0.03750449, 0.03753631, 0.03754773, 0.03755782, 0.03756427,\n",
       "       0.03758706, 0.03758983, 0.03760975, 0.03764346, 0.03767927,\n",
       "       0.03775273, 0.03783595, 0.03787188, 0.03787382, 0.03789005,\n",
       "       0.03789832, 0.03790694, 0.03791424, 0.03793115, 0.03793599,\n",
       "       0.03794077, 0.03794807, 0.03795134, 0.03795475, 0.03797355,\n",
       "       0.03798193, 0.03800453, 0.03800852, 0.03804975, 0.03805307,\n",
       "       0.03807831, 0.03809577, 0.03810337, 0.03812015, 0.03812743,\n",
       "       0.0381688 , 0.03817765, 0.03819558, 0.03819816, 0.03820088,\n",
       "       0.03823702, 0.03825469, 0.03827404, 0.03828043, 0.03828297,\n",
       "       0.03830098, 0.03830104, 0.03830355, 0.03833963, 0.03836301,\n",
       "       0.03836539, 0.03838604, 0.03840164, 0.03840671, 0.03842359,\n",
       "       0.03843526, 0.03843848, 0.0384459 , 0.03845245, 0.03846668,\n",
       "       0.03846885, 0.03847696, 0.0384896 , 0.03849416, 0.03851037,\n",
       "       0.03853116, 0.03855197, 0.0385728 , 0.03859366, 0.03861453,\n",
       "       0.03861712, 0.03863542, 0.03871862, 0.03875576, 0.03884281,\n",
       "       0.03888007, 0.03889367, 0.03916821, 0.0392147 , 0.03926101,\n",
       "       0.03929822, 0.03941015, 0.03941142, 0.03948503, 0.03948994,\n",
       "       0.03956871, 0.03964773, 0.03979934, 0.03981919, 0.0398538 ,\n",
       "       0.0398623 , 0.03991944, 0.03997853, 0.04000733, 0.04027224,\n",
       "       0.04028426, 0.04029388, 0.04031185, 0.04034781, 0.04037873,\n",
       "       0.04050335, 0.04055689, 0.04057475, 0.04071795, 0.04089773,\n",
       "       0.04106268, 0.04114449, 0.04114946, 0.04122656, 0.04124006,\n",
       "       0.04124221, 0.04124711, 0.041279  , 0.04130889, 0.04132951,\n",
       "       0.04141217, 0.0414536 , 0.04145408, 0.04151313, 0.04156859,\n",
       "       0.04157423, 0.04158378, 0.04161998, 0.0416229 , 0.04163882,\n",
       "       0.04166174, 0.04166576, 0.04172674, 0.04182496, 0.04185252,\n",
       "       0.04187118, 0.04188986, 0.04192723, 0.04193702, 0.04194165,\n",
       "       0.0419539 , 0.04198251, 0.0420021 , 0.04204284, 0.04204626,\n",
       "       0.04206276, 0.04207153, 0.04211718, 0.04213351, 0.0421426 ,\n",
       "       0.04218234, 0.04220882, 0.04232209, 0.04232467, 0.04237953,\n",
       "       0.04240584, 0.04243114, 0.04243659, 0.04247368, 0.04249092,\n",
       "       0.04249268, 0.04251105, 0.04253119, 0.04253168, 0.04254052,\n",
       "       0.04256756, 0.04257199, 0.04259216, 0.04262382, 0.04263211,\n",
       "       0.0426426 , 0.04265233, 0.04269282, 0.04269325, 0.04273337,\n",
       "       0.04277436, 0.0427931 , 0.04283535, 0.04289612, 0.04289646,\n",
       "       0.04292289, 0.04293727, 0.04298203, 0.04303534, 0.04310216,\n",
       "       0.04310306, 0.04316385, 0.04319882, 0.04320518, 0.04320527,\n",
       "       0.04323113, 0.04324157, 0.04326754, 0.04328053, 0.04328293,\n",
       "       0.04328499, 0.04330888, 0.04333486, 0.04336087, 0.04336686,\n",
       "       0.04338692, 0.04341048, 0.04341299, 0.0434391 , 0.04343993,\n",
       "       0.04347516, 0.04388052, 0.0439111 , 0.04397713, 0.04404761,\n",
       "       0.04408606, 0.04414711, 0.0442325 , 0.0442898 , 0.04435775,\n",
       "       0.04438538, 0.04438539, 0.04439692, 0.0444362 , 0.04445732,\n",
       "       0.04445835, 0.04447556, 0.04450002, 0.04451502, 0.04454745,\n",
       "       0.04455458, 0.04455623, 0.04456731, 0.04458121, 0.04458719,\n",
       "       0.04461032, 0.04462697, 0.04463397, 0.0446441 , 0.04466373,\n",
       "       0.04467381, 0.04470666, 0.04472734, 0.04475826, 0.04476982,\n",
       "       0.0448372 , 0.04485126, 0.04490571, 0.04492669, 0.04496986,\n",
       "       0.04504725, 0.04510865, 0.04520861, 0.04521981, 0.0452231 ,\n",
       "       0.04522883, 0.04523443, 0.04533402, 0.04535545, 0.04536427,\n",
       "       0.04554786, 0.04554835, 0.0456342 , 0.04564863, 0.0456726 ,\n",
       "       0.04573871, 0.04575088, 0.04575486, 0.04589608, 0.04589814,\n",
       "       0.0459141 , 0.04592007, 0.04594054, 0.04595674, 0.04596391,\n",
       "       0.04596398, 0.04597698, 0.04599678, 0.04602289, 0.0460665 ,\n",
       "       0.0460724 , 0.04607281, 0.04608419, 0.04608833, 0.04609464,\n",
       "       0.0460958 , 0.04611648, 0.04612062, 0.04613833, 0.04613925,\n",
       "       0.04622011, 0.04624157, 0.04624504, 0.04627568, 0.04628549,\n",
       "       0.04629174, 0.04633571, 0.04634296, 0.04634338, 0.04635148,\n",
       "       0.04637007, 0.04638835, 0.04639515, 0.04640177, 0.04641477,\n",
       "       0.04642025, 0.04643547, 0.04646797, 0.04647545, 0.04649203,\n",
       "       0.04651218, 0.04657777, 0.04660452, 0.04678905, 0.04679281,\n",
       "       0.04681151, 0.04681776, 0.04683864, 0.04684111, 0.04685952,\n",
       "       0.04686188, 0.04688619, 0.04693454, 0.04695795, 0.04697956,\n",
       "       0.04698509, 0.04700299, 0.04702827, 0.04705175, 0.04706931,\n",
       "       0.04707338, 0.04709689, 0.04713209, 0.04714586, 0.04719108,\n",
       "       0.04719302, 0.04721633, 0.04721758, 0.04723829, 0.04723837,\n",
       "       0.0473118 , 0.04734304, 0.04740509, 0.04741596, 0.04742789,\n",
       "       0.04742998, 0.04745168, 0.04745379, 0.04747548, 0.04751262,\n",
       "       0.0475338 , 0.04753386, 0.0475359 , 0.0476285 , 0.04772993,\n",
       "       0.0477473 , 0.04779769, 0.04781881, 0.0478457 , 0.04793143,\n",
       "       0.04806728, 0.04812607, 0.04816251, 0.0482488 , 0.04828986,\n",
       "       0.04848889, 0.04852429, 0.04861965, 0.04864588, 0.04869841,\n",
       "       0.04878279, 0.0488775 , 0.04887756, 0.04891054, 0.04892365,\n",
       "       0.04894268, 0.04894514, 0.04896587, 0.04897901, 0.04898907,\n",
       "       0.04898958, 0.04901143, 0.04901228, 0.04908017, 0.04908095,\n",
       "       0.04910531, 0.04911414, 0.04914917, 0.0491712 , 0.04918223,\n",
       "       0.04918337, 0.04921762, 0.04922195, 0.04925054, 0.04926939,\n",
       "       0.04931906, 0.04932069, 0.04932747, 0.0493534 , 0.04936245,\n",
       "       0.04938462, 0.04938969, 0.04942226, 0.04945676, 0.04949133,\n",
       "       0.04949776, 0.04952595, 0.04956062, 0.04959535, 0.04963014,\n",
       "       0.04963535, 0.0497171 , 0.04978627, 0.04985322, 0.04999421,\n",
       "       0.05028344, 0.05035505, 0.05036137, 0.05047861, 0.05050112,\n",
       "       0.05055336, 0.05056873, 0.0506338 , 0.05065903, 0.05068873,\n",
       "       0.05073468, 0.05081752, 0.05082893, 0.05088264, 0.05089103,\n",
       "       0.05090675, 0.05090835, 0.05093087, 0.05096283, 0.05098741,\n",
       "       0.05102754, 0.05104497, 0.05105175, 0.05105891, 0.05109406,\n",
       "       0.05109967, 0.05112015, 0.05114875, 0.05115652, 0.05116184,\n",
       "       0.05116882, 0.05119637, 0.05122167, 0.05122781, 0.05123672,\n",
       "       0.05125072, 0.05129475, 0.051321  , 0.05133449, 0.05136865,\n",
       "       0.05137668, 0.05139828, 0.05140245, 0.0514042 , 0.05141699,\n",
       "       0.05142823, 0.05145404, 0.05147546, 0.05154695, 0.05157404,\n",
       "       0.05173321, 0.0517628 , 0.05178366, 0.05178701, 0.0518161 ,\n",
       "       0.05183521, 0.05186461, 0.0518715 , 0.0518754 , 0.0519109 ,\n",
       "       0.05194196, 0.05200731, 0.05200768, 0.0520666 , 0.05209785,\n",
       "       0.05209955, 0.05212914, 0.05213525, 0.05215586, 0.05216047,\n",
       "       0.05219184, 0.05221172, 0.05222796, 0.05223693, 0.05227404,\n",
       "       0.05234039, 0.05236523, 0.05236856, 0.05239337, 0.05239677,\n",
       "       0.05241226, 0.05241546, 0.0524454 , 0.0525062 , 0.05252829,\n",
       "       0.05253448, 0.05258487, 0.05261947, 0.05264786, 0.05266669,\n",
       "       0.05266995, 0.05269836, 0.05270471, 0.05273318, 0.05275527,\n",
       "       0.05276167, 0.05278376, 0.0527902 , 0.05281228, 0.05284732,\n",
       "       0.05286941, 0.05289802, 0.05303616, 0.05315033, 0.05317895,\n",
       "       0.05320758, 0.05324548, 0.05329909, 0.05333255, 0.05353286,\n",
       "       0.05363301, 0.05381809, 0.05384612, 0.05387306, 0.05393658,\n",
       "       0.05394615, 0.05400543, 0.05408798, 0.05415902, 0.05418469,\n",
       "       0.05420803, 0.05422208, 0.05431326, 0.05433633, 0.05439061,\n",
       "       0.054396  , 0.05443729, 0.05444227, 0.05445596, 0.05447556,\n",
       "       0.05451989, 0.0545674 , 0.05457325, 0.05460588, 0.05463856,\n",
       "       0.05464961, 0.05483303, 0.05486068, 0.05511468, 0.05522388,\n",
       "       0.05522829, 0.05528811, 0.05534143, 0.05535805, 0.05542321,\n",
       "       0.05546145, 0.0554686 , 0.05557132, 0.05563508, 0.05576755,\n",
       "       0.05580137, 0.05580447, 0.0558278 , 0.0558613 , 0.05589033,\n",
       "       0.05589507, 0.05592985, 0.05596175, 0.05604095, 0.05606541,\n",
       "       0.05607435, 0.05621183, 0.05626041, 0.05628943, 0.05631754,\n",
       "       0.0563645 , 0.05637188, 0.05648662, 0.05652007, 0.05659635,\n",
       "       0.0567159 , 0.05674179, 0.05675526, 0.05675775, 0.05677025,\n",
       "       0.05679874, 0.05683415, 0.05684007, 0.0568495 , 0.05685866,\n",
       "       0.05687067, 0.05688307, 0.05689223, 0.05690174, 0.05691918,\n",
       "       0.05694615, 0.05698384, 0.0569927 , 0.05699336, 0.05701861,\n",
       "       0.05704648, 0.05705487, 0.05705519, 0.05707235, 0.05708127,\n",
       "       0.05708568, 0.05708596, 0.05710835, 0.05711651, 0.05713189,\n",
       "       0.05715259, 0.05717848, 0.05720519, 0.05720937, 0.0572403 ,\n",
       "       0.05727126, 0.05727842, 0.05728195, 0.05737005, 0.05738018,\n",
       "       0.05740501, 0.05748149, 0.05748637, 0.05753009, 0.05762389,\n",
       "       0.05762534, 0.05763238, 0.05765079, 0.05765769, 0.05767608,\n",
       "       0.05769034, 0.05772747, 0.05774303, 0.05778432, 0.05784911,\n",
       "       0.05790093, 0.05793734, 0.05798311, 0.05798856, 0.05798859,\n",
       "       0.057996  , 0.05803471, 0.05803997, 0.05808643, 0.05809146,\n",
       "       0.05814308, 0.05819482, 0.05820757, 0.05828018, 0.05829581,\n",
       "       0.05829868, 0.05831559, 0.05833572, 0.0583508 , 0.05838661,\n",
       "       0.05845456, 0.05852461, 0.05855801, 0.05859483, 0.05870047,\n",
       "       0.05871192, 0.05872199, 0.05894325, 0.05895713, 0.05896956,\n",
       "       0.05901037, 0.05901369, 0.05906285, 0.05912005, 0.05915487,\n",
       "       0.05916895, 0.05919027, 0.0592044 , 0.05922571, 0.0592612 ,\n",
       "       0.05927544, 0.05929672, 0.05931102, 0.05933229, 0.05933918,\n",
       "       0.05934664, 0.05936791, 0.05938057, 0.0593823 , 0.05940357,\n",
       "       0.05941801, 0.05942203, 0.05943927, 0.05946356, 0.05946417,\n",
       "       0.05947501, 0.05950516, 0.05953305, 0.05954682, 0.05963035,\n",
       "       0.05969885, 0.05979455, 0.05984822, 0.05985538, 0.05990094,\n",
       "       0.05991397, 0.05994388, 0.05995285, 0.05997998, 0.05999384,\n",
       "       0.06003491, 0.06011099, 0.06015503, 0.06018695, 0.06025909,\n",
       "       0.06031961, 0.06034424, 0.06036517, 0.06039176, 0.06039545,\n",
       "       0.06041883, 0.06042575, 0.06044614, 0.06055334, 0.06056219,\n",
       "       0.06058529, 0.06063573, 0.06064184, 0.06067457, 0.06071346,\n",
       "       0.06077294, 0.06080388, 0.06080579, 0.06080902, 0.06087537,\n",
       "       0.06090829, 0.06096257, 0.0610094 , 0.06109728, 0.0611405 ,\n",
       "       0.06119002, 0.06124828, 0.06131488, 0.06131604, 0.06139898,\n",
       "       0.06142808, 0.06143321, 0.06154467, 0.06157387, 0.06157576,\n",
       "       0.06163232, 0.06169086, 0.06176007, 0.0617658 , 0.06177881,\n",
       "       0.06182509, 0.06184744, 0.0618596 , 0.06186596, 0.06186695,\n",
       "       0.06192581, 0.06195154, 0.06202568, 0.062031  , 0.06206296,\n",
       "       0.06214926, 0.06215999, 0.06223288, 0.06224632, 0.06225932,\n",
       "       0.06228959, 0.06231887, 0.0623312 , 0.06236162, 0.0623964 ,\n",
       "       0.06239759, 0.06245531, 0.06254297, 0.0625536 , 0.06257775,\n",
       "       0.06271107, 0.06281231, 0.06284612, 0.06287326, 0.06287997,\n",
       "       0.06298169, 0.06301567, 0.06304968, 0.06308348, 0.06311779,\n",
       "       0.06312485, 0.0631519 , 0.06318604, 0.063188  , 0.06324781,\n",
       "       0.06326316, 0.06347878, 0.06349611, 0.06350528, 0.06353715,\n",
       "       0.06356375, 0.06363591, 0.06366121, 0.06377849, 0.06385782,\n",
       "       0.06397233, 0.06400644, 0.06401059, 0.06408003, 0.06408725,\n",
       "       0.06415677, 0.06416409, 0.0641648 , 0.06420258, 0.0642337 ,\n",
       "       0.06427223, 0.0642797 , 0.06431833, 0.06434944, 0.06442683,\n",
       "       0.06449406, 0.06458333, 0.06479869, 0.06485536, 0.0649853 ,\n",
       "       0.06507173, 0.06510877, 0.0651541 , 0.06518908, 0.06545939,\n",
       "       0.0655955 , 0.06576055, 0.06577387, 0.06586968, 0.06591563,\n",
       "       0.06599913, 0.06608925, 0.06610018, 0.0661465 , 0.06616603,\n",
       "       0.06632858, 0.0663517 , 0.06638829, 0.06644203, 0.0665924 ,\n",
       "       0.06667413, 0.06668165, 0.06674909, 0.06675316, 0.06676878,\n",
       "       0.06677925, 0.06680728, 0.06681212, 0.06684621, 0.0668501 ,\n",
       "       0.06685188, 0.06685686, 0.06685692, 0.0668657 , 0.06687122,\n",
       "       0.06691146, 0.06692105, 0.06692574, 0.06692887, 0.06693046,\n",
       "       0.06696987, 0.06701141, 0.06701293, 0.06702405, 0.06702533,\n",
       "       0.067031  , 0.06704144, 0.06704937, 0.06705989, 0.06707331,\n",
       "       0.06710111, 0.06710575, 0.06710951, 0.06714224, 0.06718751,\n",
       "       0.06719057, 0.06719854, 0.06720114, 0.06720154, 0.06721831,\n",
       "       0.06722094, 0.06723503, 0.06726099, 0.06727374, 0.06729102,\n",
       "       0.0673098 , 0.06732042, 0.06732743, 0.06733091, 0.06733827,\n",
       "       0.06736518, 0.06737784, 0.0673803 , 0.06738499, 0.067414  ,\n",
       "       0.06742484, 0.06744017, 0.0674609 , 0.06747866, 0.06748643,\n",
       "       0.06750019, 0.06750175, 0.06751907, 0.06755493, 0.06756155,\n",
       "       0.06761973, 0.06762088, 0.06764926, 0.0676669 , 0.06769654,\n",
       "       0.06771353, 0.06771415, 0.06774181, 0.06775576, 0.06776147,\n",
       "       0.06778955, 0.06779132, 0.06780887, 0.06783883, 0.06784814,\n",
       "       0.06785635, 0.06786271, 0.06788641, 0.06790042, 0.0679039 ,\n",
       "       0.06795153, 0.06795478, 0.06796316, 0.06805414, 0.068057  ,\n",
       "       0.06809017, 0.06812891, 0.0682019 , 0.06823032, 0.06830989,\n",
       "       0.06834565, 0.06834645, 0.06835206, 0.06837283, 0.06840121,\n",
       "       0.06842048, 0.06843666, 0.06844777, 0.06845206, 0.06846821,\n",
       "       0.06847099, 0.0685112 , 0.06853316, 0.06856391, 0.06860172,\n",
       "       0.06862876, 0.06869051, 0.06879306, 0.06942325, 0.06950181,\n",
       "       0.06950825, 0.06954997, 0.06956614, 0.0696114 , 0.06961466,\n",
       "       0.06964938, 0.06968367, 0.0697347 , 0.06973838, 0.06974644,\n",
       "       0.06979657, 0.06985859, 0.06992076, 0.06996285, 0.06998308,\n",
       "       0.07004297, 0.07004555, 0.07010817, 0.07011321, 0.07015089,\n",
       "       0.07026414, 0.07026725, 0.07048411, 0.07049163, 0.07052855,\n",
       "       0.07056775, 0.07064402, 0.07068221, 0.07077856, 0.07086338,\n",
       "       0.07086742, 0.0709484 , 0.070957  , 0.07100046, 0.07103362,\n",
       "       0.07107631, 0.07111905, 0.07136735, 0.07161233, 0.07168828,\n",
       "       0.07192386, 0.07221976, 0.07232025, 0.0724193 , 0.07242091,\n",
       "       0.07242557, 0.07252205, 0.07255457, 0.07262071, 0.0727126 ,\n",
       "       0.07271452, 0.07272213, 0.07272576, 0.07282402, 0.07282834,\n",
       "       0.07286572, 0.07286592, 0.07290374, 0.07291628, 0.0729264 ,\n",
       "       0.07294477, 0.07299539, 0.07301543, 0.07302927, 0.07303192,\n",
       "       0.07304608, 0.07309623, 0.07309686, 0.07311933, 0.07313263,\n",
       "       0.0731607 , 0.07320823, 0.07323649, 0.07325809, 0.07329009,\n",
       "       0.07335195, 0.07335502, 0.07337496, 0.07342011, 0.07342633,\n",
       "       0.07345456, 0.07347779, 0.07348535, 0.07352933, 0.0735575 ,\n",
       "       0.0735616 , 0.0736091 , 0.07385109, 0.07399115, 0.07413931,\n",
       "       0.07416257, 0.07435718, 0.07446067, 0.0747105 , 0.07486101,\n",
       "       0.07505732, 0.07514904, 0.07519149, 0.07524498, 0.07524601,\n",
       "       0.07527747, 0.07533607, 0.07546348, 0.07556217, 0.07565299,\n",
       "       0.07569613, 0.07579601, 0.07590117, 0.07596266, 0.07600364,\n",
       "       0.07604465, 0.07605597, 0.07606945, 0.07611296, 0.07612681,\n",
       "       0.07617669, 0.07620912, 0.07621961, 0.07629159, 0.07632676,\n",
       "       0.07643442, 0.07644129, 0.07644624, 0.07656377, 0.07693306,\n",
       "       0.07707403, 0.07717714, 0.07720422, 0.0773058 , 0.0773277 ,\n",
       "       0.07742569, 0.0774411 , 0.07752166, 0.0775468 , 0.07755623,\n",
       "       0.07765458, 0.07765736, 0.07766192, 0.07771007, 0.07775433,\n",
       "       0.07776018, 0.07779091, 0.07779873, 0.07783681, 0.0779119 ,\n",
       "       0.07791485, 0.07797549, 0.07808882, 0.07809251, 0.07810048,\n",
       "       0.07813096, 0.07813253, 0.07821537, 0.07822104, 0.0782703 ,\n",
       "       0.07860414, 0.07861096, 0.07861762, 0.07868758, 0.07869452,\n",
       "       0.07870128, 0.07875771, 0.07882801, 0.07883288, 0.07884373,\n",
       "       0.07884674, 0.07890136, 0.0789023 , 0.07893929, 0.07897066,\n",
       "       0.07900808, 0.07904164, 0.07910968, 0.07910981, 0.07911156,\n",
       "       0.07918294, 0.07923753, 0.07925393, 0.0792645 , 0.07927037,\n",
       "       0.07931949, 0.07932232, 0.07932541, 0.07936116, 0.07936547,\n",
       "       0.07938976, 0.07940264, 0.07941311, 0.07946081, 0.07946591,\n",
       "       0.0795308 , 0.07960158, 0.07960709, 0.07967253, 0.07967794,\n",
       "       0.07969228, 0.07974364, 0.07974895, 0.07975546, 0.07981493,\n",
       "       0.07982014, 0.07989149, 0.07991229, 0.07996982, 0.08004531,\n",
       "       0.08005278, 0.08007429, 0.08012328, 0.08026478, 0.08033578,\n",
       "       0.08041031, 0.0804395 , 0.0805498 , 0.08063362, 0.08080203,\n",
       "       0.08085835, 0.0808677 , 0.08091476, 0.08097126, 0.08097601,\n",
       "       0.08102785, 0.08105208, 0.08108453, 0.0811244 , 0.0811413 ,\n",
       "       0.08119816, 0.08125511, 0.08179272, 0.08233843, 0.08262923,\n",
       "       0.08266839, 0.08282837, 0.08286319, 0.08290924, 0.08294107,\n",
       "       0.08295273, 0.08306184, 0.08324685, 0.08331115, 0.08333822,\n",
       "       0.08339708, 0.0835124 , 0.08357478, 0.08372313, 0.08374467,\n",
       "       0.08375315, 0.08376772, 0.08388318, 0.08397912, 0.08406757,\n",
       "       0.08410441, 0.08423279, 0.08437299, 0.08441681, 0.08454778,\n",
       "       0.08458735, 0.08465236, 0.08470667, 0.08489014, 0.08500989,\n",
       "       0.0852358 , 0.08526447, 0.08531546, 0.08532827, 0.08536652,\n",
       "       0.08546881, 0.08555258, 0.08557136, 0.08561011, 0.08562272,\n",
       "       0.0856237 , 0.08567415, 0.08576108, 0.08576613, 0.08577846,\n",
       "       0.0858257 , 0.08583683, 0.08583989, 0.0858562 , 0.08588224,\n",
       "       0.08591276, 0.08592995, 0.08594796, 0.08600596, 0.08606196,\n",
       "       0.0860649 , 0.08608215, 0.08612494, 0.08614163, 0.08616522,\n",
       "       0.08621829, 0.08623509, 0.08624531, 0.08630564, 0.08636607,\n",
       "       0.08637216, 0.08638876, 0.08641613, 0.08642659, 0.08644938,\n",
       "       0.08648721, 0.08654317, 0.08682802, 0.08684139, 0.08696201,\n",
       "       0.08702921, 0.08713211, 0.0872914 , 0.08737524, 0.08744692,\n",
       "       0.08749767, 0.08762067, 0.08763379, 0.08774427, 0.08786846,\n",
       "       0.0879718 , 0.08799324, 0.08802464, 0.08821406, 0.08844732,\n",
       "       0.08884066, 0.08896494, 0.08902057, 0.08907426, 0.08908962,\n",
       "       0.0893266 , 0.08936912, 0.0896087 , 0.08963257, 0.08964547,\n",
       "       0.08965305, 0.08968931, 0.08969776, 0.08970771, 0.08974799,\n",
       "       0.09000325, 0.09000925, 0.09015011, 0.09024991, 0.09025758,\n",
       "       0.09035034, 0.09036655, 0.0903773 , 0.09042083, 0.09045111,\n",
       "       0.09057741, 0.09064071, 0.09064336, 0.09076762, 0.09077895,\n",
       "       0.09080202, 0.09083123, 0.090849  , 0.09089494, 0.09090248,\n",
       "       0.09114018, 0.09120195, 0.09128251, 0.09144026, 0.0914442 ,\n",
       "       0.09152533, 0.09152865, 0.09191138, 0.09204016, 0.09208169,\n",
       "       0.09213737, 0.09216956, 0.09229944, 0.09241518, 0.09248333,\n",
       "       0.09248822, 0.09266344, 0.0927368 , 0.09289923, 0.09307137,\n",
       "       0.093137  , 0.09325702, 0.09330352, 0.09345311, 0.09367689,\n",
       "       0.09370155, 0.09395059, 0.09401556, 0.09403419, 0.09420202,\n",
       "       0.09427643, 0.09467084, 0.09468855, 0.09477206, 0.09486025,\n",
       "       0.09513563, 0.09519261, 0.09522656, 0.09527732, 0.09544736,\n",
       "       0.09553269, 0.09553748, 0.09570025, 0.09605322, 0.09608055,\n",
       "       0.0963212 , 0.09645445, 0.09658734, 0.0967125 , 0.09672134,\n",
       "       0.09675625, 0.09678254, 0.09691636, 0.09710378, 0.09712712,\n",
       "       0.09723592, 0.09725118, 0.09726367, 0.09727086, 0.09738808,\n",
       "       0.09740087, 0.09744891, 0.09750491, 0.09753872, 0.09759185,\n",
       "       0.09763595, 0.09765649, 0.09766   , 0.09772825, 0.09779188,\n",
       "       0.09779661, 0.09786508, 0.09793366, 0.09800236, 0.09816601,\n",
       "       0.09817801, 0.09833054, 0.09851626, 0.09854426, 0.09860675,\n",
       "       0.09866238, 0.09879881, 0.0988856 , 0.09890884, 0.09901659,\n",
       "       0.09905848, 0.09915351, 0.0991641 , 0.09921194, 0.09924806,\n",
       "       0.09936856, 0.09948344, 0.09955157, 0.09956812, 0.09961208,\n",
       "       0.0996442 , 0.09973509, 0.09982152, 0.09989282, 0.09998865,\n",
       "       0.10003421, 0.10013016, 0.10017627, 0.10018914, 0.10023345,\n",
       "       0.10026931, 0.10027236, 0.10032768, 0.10034904, 0.10043218,\n",
       "       0.1004461 , 0.10055797, 0.10056647, 0.10072364, 0.10074721,\n",
       "       0.10078371, 0.10084146, 0.10087453, 0.10088837, 0.10096009,\n",
       "       0.10096363, 0.10100377, 0.10105294, 0.1012322 , 0.10132215,\n",
       "       0.10142609, 0.10142747, 0.101572  , 0.10159331, 0.10168413,\n",
       "       0.10171906, 0.10180933, 0.10195373, 0.10208597, 0.10246917,\n",
       "       0.10275349, 0.10278376, 0.10314968, 0.10316846, 0.10317231,\n",
       "       0.10349133, 0.10363579, 0.1037997 , 0.10389217, 0.10392307,\n",
       "       0.10407458, 0.10416664, 0.10441265, 0.10450578, 0.1045371 ,\n",
       "       0.10455275, 0.10472366, 0.1047924 , 0.10480131, 0.10514889,\n",
       "       0.10537077, 0.10542986, 0.10547981, 0.10598124, 0.10607861,\n",
       "       0.10616056, 0.10620299, 0.1062851 , 0.10639139, 0.10648593,\n",
       "       0.10656662, 0.10667568, 0.1067709 , 0.1068403 , 0.10686635,\n",
       "       0.10705394, 0.10714436, 0.10759091, 0.10774116, 0.1080438 ,\n",
       "       0.10807131, 0.10816102, 0.10816379, 0.1081962 , 0.10831358,\n",
       "       0.10834934, 0.10835797, 0.10846688, 0.10850321, 0.10862092,\n",
       "       0.10887252, 0.10896799, 0.10963147, 0.11092584, 0.11110537,\n",
       "       0.11127714, 0.11130634, 0.11167408, 0.11173731, 0.11185149,\n",
       "       0.11205074, 0.11257986, 0.11287933, 0.11294569, 0.11295524,\n",
       "       0.11302288, 0.11327932, 0.11394183, 0.11410404, 0.11410909,\n",
       "       0.11425769, 0.11439356, 0.11450741, 0.11459379, 0.11469426,\n",
       "       0.11473704, 0.11481246, 0.11501706, 0.11506042, 0.11511972,\n",
       "       0.11514795, 0.11522264, 0.11550662, 0.1156271 , 0.11619191,\n",
       "       0.11621529, 0.11645843, 0.11665835, 0.11666118, 0.11668375,\n",
       "       0.11669107, 0.11674052, 0.11685725, 0.11716179, 0.11722419,\n",
       "       0.1174817 , 0.11754497, 0.11760729, 0.11764895, 0.11782093,\n",
       "       0.11788949, 0.11796238, 0.11817147, 0.11827806, 0.11830665,\n",
       "       0.11873031, 0.11897562, 0.11906161, 0.1192527 , 0.11939605,\n",
       "       0.11956447, 0.11970086, 0.11973369, 0.1197644 , 0.11982356,\n",
       "       0.11990373, 0.11999133, 0.12023286, 0.1205319 , 0.1206382 ,\n",
       "       0.12064161, 0.12074476, 0.12128165, 0.12129231, 0.1215182 ,\n",
       "       0.12153463, 0.12182169, 0.12224454, 0.12341536, 0.12365012,\n",
       "       0.12373773, 0.12375184, 0.12411962, 0.12452788, 0.12465795,\n",
       "       0.12474085, 0.12479807, 0.1250578 , 0.12585266, 0.12598273,\n",
       "       0.12602613, 0.12604313, 0.12672811, 0.12708409, 0.12726335,\n",
       "       0.12744347, 0.12762446, 0.12765291, 0.12768434, 0.12770821,\n",
       "       0.12811984, 0.12858271, 0.12917452, 0.12941448, 0.12973051,\n",
       "       0.13014837, 0.13018878, 0.13094344, 0.13102175, 0.13105247,\n",
       "       0.13106965, 0.13118465, 0.13130367, 0.13133595, 0.13142841,\n",
       "       0.13152026, 0.13154569, 0.13273903, 0.13337329, 0.13374918,\n",
       "       0.13393848, 0.1341287 , 0.13465005, 0.13465062, 0.13469859,\n",
       "       0.13497641, 0.13548729, 0.13575636, 0.135872  , 0.13613681,\n",
       "       0.13699532, 0.13754815, 0.13791459, 0.13803732, 0.13930293,\n",
       "       0.13933014, 0.13943379, 0.13945088, 0.13958805, 0.14008926,\n",
       "       0.14033359, 0.14039631, 0.14067389, 0.1407202 , 0.14079777,\n",
       "       0.14088528, 0.14114706, 0.14129625, 0.14167324, 0.1417334 ,\n",
       "       0.14204167, 0.14210629, 0.14262797, 0.14282641, 0.14284787,\n",
       "       0.14340078, 0.14342621, 0.14379665, 0.14380224, 0.1439955 ,\n",
       "       0.14416764, 0.14420754, 0.14431669, 0.14439597, 0.14441164,\n",
       "       0.14444512, 0.14461672, 0.14467729, 0.14480022, 0.14499764,\n",
       "       0.14520833, 0.14532122, 0.14542629, 0.14560896, 0.14612498,\n",
       "       0.14621711, 0.14656639, 0.14672172, 0.14692029, 0.14692639,\n",
       "       0.1473741 , 0.1475264 , 0.14754077, 0.14816391, 0.14837361,\n",
       "       0.14883763, 0.1491352 , 0.14965489, 0.14996167, 0.15003077,\n",
       "       0.15055183, 0.15092599, 0.15113538, 0.15124638, 0.1515571 ,\n",
       "       0.15176945, 0.15198282, 0.15200381, 0.15204137, 0.15232408,\n",
       "       0.15241261, 0.15261947, 0.1527557 , 0.15294754, 0.15304264,\n",
       "       0.15305354, 0.15308365, 0.15319437, 0.15353812, 0.1547004 ,\n",
       "       0.15472433, 0.15483512, 0.15542448, 0.15547715, 0.15648367,\n",
       "       0.15651959, 0.15688928, 0.15697608, 0.15702604, 0.15739045,\n",
       "       0.15784921, 0.15830782, 0.1588745 , 0.15922343, 0.15974804,\n",
       "       0.16008766, 0.16009635, 0.16028243, 0.16063196, 0.16237748,\n",
       "       0.16246846, 0.1640319 , 0.16421528, 0.16424429, 0.16446385,\n",
       "       0.16491968, 0.16549577, 0.16550733, 0.16566508, 0.16626811,\n",
       "       0.16635572, 0.16646456, 0.16686328, 0.1674508 , 0.16802686,\n",
       "       0.16813969, 0.16838049, 0.1686231 , 0.16868054, 0.16875942,\n",
       "       0.16903921, 0.16909422, 0.16937239, 0.16948619, 0.16978488,\n",
       "       0.17013925, 0.17021307, 0.17062908, 0.17095377, 0.17133061,\n",
       "       0.17215391, 0.17216194, 0.17235766, 0.17242939, 0.17245402,\n",
       "       0.17249589, 0.17249815, 0.17382273, 0.17408011, 0.17443071,\n",
       "       0.17456168, 0.17480791, 0.17485141, 0.17509414, 0.17510029,\n",
       "       0.17631439, 0.17632126, 0.17681132, 0.17698504, 0.17706156,\n",
       "       0.17743321, 0.17749034, 0.17796387, 0.17801663, 0.17855811,\n",
       "       0.17932706, 0.179531  , 0.17969572, 0.17975164, 0.18109199,\n",
       "       0.18117197, 0.18126563, 0.18152219, 0.18194603, 0.18203194,\n",
       "       0.18275713, 0.18359447, 0.18392276, 0.18425167, 0.1845166 ,\n",
       "       0.18454555, 0.184809  , 0.18494094, 0.1859657 , 0.18619975,\n",
       "       0.18732285, 0.18860248, 0.18900765, 0.18908112, 0.18928709,\n",
       "       0.18948126, 0.18982735, 0.19000998, 0.19027621, 0.19048311,\n",
       "       0.19050167, 0.19054371, 0.19054839, 0.19078557, 0.19079635,\n",
       "       0.19108254, 0.1911471 , 0.19118515, 0.19127794, 0.19154339,\n",
       "       0.19181297, 0.19183562, 0.19196088, 0.19209962, 0.19226106,\n",
       "       0.19237811, 0.1939736 , 0.19407093, 0.19424357, 0.19499668,\n",
       "       0.19507229, 0.19556254, 0.19558577, 0.19567993, 0.19679406,\n",
       "       0.1969666 , 0.19745378, 0.19797589, 0.19853793, 0.19858879,\n",
       "       0.19928406, 0.20005143, 0.20164006, 0.20220809, 0.20324984,\n",
       "       0.20399955, 0.20402061, 0.20410827, 0.20469431, 0.20502598,\n",
       "       0.20560189, 0.20618331, 0.20650266, 0.20684737, 0.20717775,\n",
       "       0.20748831, 0.20752087, 0.20799541, 0.208153  , 0.20938101,\n",
       "       0.21075813, 0.21110806, 0.21178974, 0.21181002, 0.21260351,\n",
       "       0.2126211 , 0.21278962, 0.21346023, 0.2139896 , 0.21402318,\n",
       "       0.21413684, 0.21430495, 0.21503577, 0.2159416 , 0.21712572,\n",
       "       0.21746327, 0.2175079 , 0.21880491, 0.21937681, 0.22014864,\n",
       "       0.22043084, 0.22194043, 0.22211896, 0.22237121, 0.22269924,\n",
       "       0.22275274, 0.22350869, 0.22484471, 0.22584854, 0.22621118,\n",
       "       0.22679112, 0.22682686, 0.22708804, 0.22752578, 0.22772747,\n",
       "       0.22787217, 0.2287715 , 0.22914141, 0.23069339, 0.23103097,\n",
       "       0.23162731, 0.231671  , 0.23214957, 0.23215748, 0.23315726,\n",
       "       0.23422119, 0.23571129, 0.23583437, 0.23762893, 0.24000646,\n",
       "       0.24126937, 0.24141385, 0.24180702, 0.24290574, 0.24335188,\n",
       "       0.24357861, 0.24399008, 0.24774   , 0.24799412, 0.2482332 ,\n",
       "       0.24940083, 0.24969534, 0.25110757, 0.25479814, 0.25510802,\n",
       "       0.25536681, 0.25576459, 0.2559997 , 0.25915157, 0.26013867,\n",
       "       0.26015634, 0.26104512, 0.2652063 , 0.26536911, 0.26578246,\n",
       "       0.26635845, 0.26765055, 0.26771374, 0.26793977, 0.2714081 ,\n",
       "       0.2714105 , 0.27181851, 0.27358695, 0.27360956, 0.2742552 ,\n",
       "       0.2780129 , 0.27938589, 0.27990187, 0.28017349, 0.28175996,\n",
       "       0.28612794, 0.28726656, 0.28923845, 0.29023643, 0.29058875,\n",
       "       0.29074683, 0.29166651, 0.2933525 , 0.29546833, 0.29577182,\n",
       "       0.29786204, 0.29827404, 0.29851832, 0.29927843, 0.30098314,\n",
       "       0.30320238, 0.3034046 , 0.30516483, 0.30696152, 0.30724581,\n",
       "       0.30756794, 0.30873212, 0.31116477, 0.31137619, 0.31222221,\n",
       "       0.31437678, 0.31472665, 0.31507832, 0.31624507, 0.31659548,\n",
       "       0.31922736, 0.31985241, 0.32113262, 0.32253965, 0.32373817,\n",
       "       0.32547609, 0.32592223, 0.32639189, 0.32700406, 0.32705397,\n",
       "       0.32782597, 0.33046546, 0.33181651, 0.33206223, 0.3325469 ,\n",
       "       0.33399605, 0.33537766, 0.33659406, 0.33704919, 0.33876633,\n",
       "       0.34043533, 0.34237825, 0.34312505, 0.34397478, 0.34605461,\n",
       "       0.34715748, 0.3500133 , 0.35018719, 0.35143888, 0.35202235,\n",
       "       0.35273769, 0.35431535, 0.35640012, 0.35761886, 0.35776323,\n",
       "       0.35796521, 0.35803507, 0.35974623, 0.36100568, 0.36165248,\n",
       "       0.36165302, 0.36194453, 0.36269921, 0.36404251, 0.36610841,\n",
       "       0.36673721, 0.36815647, 0.3714427 , 0.37186828, 0.37205871,\n",
       "       0.37363142, 0.37544915, 0.37694124, 0.38093584, 0.38287014,\n",
       "       0.38291088, 0.38294735, 0.38314142, 0.38533852, 0.38555752,\n",
       "       0.38571908, 0.38602661, 0.38643375, 0.38647929, 0.38713455,\n",
       "       0.38858314, 0.38982286, 0.38984837, 0.38987513, 0.38997903,\n",
       "       0.39070574, 0.39087634, 0.39182392, 0.39246385, 0.39287778,\n",
       "       0.39314108, 0.39372138, 0.39490873, 0.39733546, 0.39741724,\n",
       "       0.39813253, 0.40043159, 0.400767  , 0.40219835, 0.40634985,\n",
       "       0.40696262, 0.41021357, 0.41037665, 0.41088143, 0.41452021,\n",
       "       0.41485657, 0.41589288, 0.41712498, 0.41938939, 0.42012213,\n",
       "       0.42170408, 0.42371583, 0.42541122, 0.42678257, 0.42936766,\n",
       "       0.42973228, 0.42995485, 0.43098639, 0.43217747, 0.4327991 ,\n",
       "       0.43308228, 0.43309042, 0.43710764, 0.43802166, 0.43874075,\n",
       "       0.44015503, 0.44238724, 0.44579822, 0.4470449 , 0.44790793,\n",
       "       0.44812376, 0.45259959, 0.45388207, 0.4554405 , 0.45566827,\n",
       "       0.45634676, 0.45838741, 0.45971482, 0.46151649, 0.4622946 ,\n",
       "       0.4640687 , 0.46563913, 0.46781913, 0.4688121 , 0.47120607,\n",
       "       0.47206376, 0.47305797, 0.47325732, 0.47603023, 0.47978823,\n",
       "       0.48014605, 0.48351964, 0.48413667, 0.48512459, 0.48531481,\n",
       "       0.48717469, 0.48836738, 0.48947865, 0.48971481, 0.49241646,\n",
       "       0.49482157, 0.49624933, 0.49672811, 0.49734412, 0.49880582,\n",
       "       0.49887089, 0.50689601, 0.50880632, 0.51202448, 0.51705688,\n",
       "       0.52012614, 0.53417677, 0.53493281, 0.5365639 , 0.53695528,\n",
       "       0.54001608, 0.54116257, 0.54251328, 0.54452548, 0.54459524,\n",
       "       0.54767184, 0.54873772, 0.55706191, 0.56592613, 0.57803516,\n",
       "       0.57804794, 0.57929279, 0.58223699, 0.58325495, 0.58762938,\n",
       "       0.59149325, 0.59243991, 0.59599034, 0.59873668, 0.60022325,\n",
       "       0.6019003 , 0.6069739 , 0.60963474, 0.61365924, 0.62214583,\n",
       "       0.6238232 , 0.6248521 , 0.63887511, 0.64012311, 0.64353508,\n",
       "       0.64413448, 0.64423066, 0.64724214, 0.65476168, 0.65546364,\n",
       "       0.65981897, 0.6602064 , 0.66121505, 0.66494209, 0.66716782,\n",
       "       0.66807767, 0.66823001, 0.67835342, 0.68141129, 0.68557831,\n",
       "       0.68982088, 0.69098924, 0.69343455, 0.69599426, 0.69878246,\n",
       "       0.69879101, 0.7047612 , 0.70588889, 0.70730136, 0.71532589,\n",
       "       0.71684002, 0.71737492, 0.71804734, 0.71960428, 0.74395474,\n",
       "       0.74409154, 0.76931047, 0.78156549, 0.78767085, 0.79786676,\n",
       "       0.80561464, 0.81228915, 0.8387443 , 0.84004722, 0.84091799,\n",
       "       0.84526878, 0.87475211, 0.88787646, 0.89243941, 0.8976794 ,\n",
       "       0.90757409, 0.92086773, 0.92261628, 0.95213874, 0.9601789 ,\n",
       "       0.9634985 , 0.96557127, 0.9700499 , 1.        ])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numpy.unique(vectorized_text.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "77206ca20a93ef1ec99f9edfda917eaa4dad77649573d0f1a3f3cc6e9621e6de"
  },
  "kernelspec": {
   "display_name": "Python 3.7.11 ('capstone')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}