twitcrime1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\Anjali\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\Anjali\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import  WordNetLemmatizer\n",
    "lemmatizer =  WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import wordnet\n",
    "from nltk import pos_tag\n",
    "def get_simple_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN      \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['!',\n",
       " '\"',\n",
       " '#',\n",
       " '$',\n",
       " '%',\n",
       " '&',\n",
       " \"'\",\n",
       " '(',\n",
       " ')',\n",
       " '*',\n",
       " '+',\n",
       " ',',\n",
       " '-',\n",
       " '.',\n",
       " '/',\n",
       " ':',\n",
       " ';',\n",
       " '<',\n",
       " '=',\n",
       " '>',\n",
       " '?',\n",
       " '@',\n",
       " '[',\n",
       " '\\\\',\n",
       " ']',\n",
       " '^',\n",
       " '_',\n",
       " '`',\n",
       " '{',\n",
       " '|',\n",
       " '}',\n",
       " '~']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "import string \n",
    "#stops = set(stopwords.words('english'))\n",
    "stops = list(string.punctuation)\n",
    "stops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positive tweets percentage: 29.26829268292683 %\n",
      "Negative tweets percentage: 22.172949002217294 %\n",
      "\n",
      "\n",
      "Positive tweets:\n",
      "@djcarlile @JPFinlayNBCS The defense looks for real though and they've faced some good QBs so far and have a 5-2 re… https://t.co/XR563Rbfz2\n",
      "RT @5liveSport: On Sports Extra, more #CarabaoCup action tonight\n",
      "\n",
      "⚽ @NorwichCityFC eliminated @afcbournemouth on the only other occasion th…\n",
      "On Sports Extra, more #CarabaoCup action tonight\n",
      "\n",
      "⚽ @NorwichCityFC eliminated @afcbournemouth on the only other occ… https://t.co/XAww6STLtf\n",
      "@nytimes Anything the POTUS has done has faced legal opposition, even though most have failed.  The left has made t… https://t.co/83HvxNm7R5\n",
      "@SuzanneWaldman This \"conflict &amp; drama\" is barely half a decade old, and largely coincides with the Arab Spring and… https://t.co/GrbJVV4xsS\n",
      "2.25: Due to the long delay of my luggage, I faced much inconveniences. Was really reluctant to spend just cos of i… https://t.co/BDPneuZLAm\n",
      "The IN Democratic Party is endorsing the Libertarian candidate rather than their own candidate to get the 2-faced… https://t.co/81cbZEetng\n",
      "RT @Handsome_Jake_: “The 98 Yankees only lost 2 playoff games!”\n",
      "\n",
      "Do you know who they faced? \n",
      "\n",
      "88 win Rangers team\n",
      "89 win Cleveland team\n",
      "98…\n",
      "In 1973, in 2 rounds, Foreman defeated Joe Frazier to become the new undisputed heavyweight champion. In 2 rounds,… https://t.co/QadeKXUGiM\n",
      "@SECNetwork @fanslsu @LSUfootball Alabama’s defense isn’t nothing new. We’ve faced 2 of the same caliber defenses a… https://t.co/DfUGMoPn6g\n",
      "\n",
      "\n",
      "Negative tweets:\n",
      "@tfwrail 2/2 same managerial team that ran down the ariva trains Wales service and are now faced with the task of r… https://t.co/zaHiwRtRwE\n",
      "Shrewsbury accident and emergency department is full of 2 faced lying nurses that deliberately talks bullshit behind patients backs\n",
      "In other news, I tried playing Rivals last night and I basically faced teams with 2+ icons every single game.\n",
      "\n",
      "Is e… https://t.co/EWWyGfgDKd\n",
      "It don’t get more bipolar than a Aquarius. I lie. Them gems 2 faced bad\n",
      "The attacks on the #TreeOfLife synagogue were a particularly harsh blow.  Jews, of course, have faced a particularl… https://t.co/EqSAC5iDRm\n",
      "'Nowhere to hide': Ex-PM John Major's warning to hard line Brexiteers\n",
      "https://t.co/fUAlTMhyqF Major a 2 faced scare… https://t.co/dkKzkk0vHP\n",
      "I hate people lmaoo 2 mf faced\n",
      "RT @jordanscoble1: 2 faced people are the worst🤢\n",
      "@manishm28301746 Sorry to know about the issue you've faced in regard to the delivery of your order. We'd like to t… https://t.co/7XDWM9nhuI\n",
      "RT @yourmagicshopp: You aren’t the same person you were 2 or 3 years ago. you’ve grown an unbelievable amount since then. you are not the s…\n"
     ]
    }
   ],
   "source": [
    "import re \n",
    "import tweepy \n",
    "from tweepy import OAuthHandler \n",
    "from textblob import TextBlob \n",
    "import csv\n",
    "import string \n",
    "from nltk.corpus import stopwords\n",
    "class TwitterClient(object): \n",
    "\n",
    "    def __init__(self): \n",
    "        consumer_key = \"rXBTXz6yE6VjactwIFfQEHLwX\"\n",
    "        consumer_secret = \"3wluIGBoyitUyr8m0HRhiap6UhUFkGskWJ2DA0J7v4es9Dfhwo\"\n",
    "        access_token = \"1055057699078107137-ntZ1ReqvsUvpPDJpKjUaV5oc0m8zhb\"\n",
    "        access_token_secret = \"39Ew12bFgpyITIjo9GOhvD2R3mlpwvrbvAaCE5XmGQYm4\"\n",
    "\n",
    "\n",
    "        try: \n",
    "            # create OAuthHandler object \n",
    "            self.auth = OAuthHandler(consumer_key, consumer_secret) \n",
    "            # set access token and secret \n",
    "            self.auth.set_access_token(access_token, access_token_secret) \n",
    "            # create tweepy API object to fetch tweets \n",
    "            self.api = tweepy.API(self.auth) \n",
    "        except: \n",
    "            print(\"Error: Authentication Failed\") \n",
    "\n",
    "    def clean_tweet(self, tweet): \n",
    "        ''' \n",
    "        Utility function to clean tweet text by removing links, special characters \n",
    "        using simple regex statements. \n",
    "        '''\n",
    "        return ' '.join(re.sub(\"(@[A-Za-z0-9]+)|([^0-9A-Za-z \\t])|(\\w+:\\/\\/\\S+)\", \" \", tweet).split())         \n",
    "        \n",
    "    def get_tweet_sentiment(self, tweet): \n",
    "        ''' \n",
    "        Utility function to classify sentiment of passed tweet \n",
    "        using textblob's sentiment method \n",
    "        '''\n",
    "        # create TextBlob object of passed tweet text \n",
    "        analysis = TextBlob(self.clean_tweet(tweet)) \n",
    "        # set sentiment \n",
    "        if analysis.sentiment.polarity > 0: \n",
    "            return 1\n",
    "        elif analysis.sentiment.polarity == 0: \n",
    "            return 0\n",
    "        else: \n",
    "            return -1\n",
    "\n",
    "    def get_tweets(self, query, count = 10): \n",
    "        ''' \n",
    "        Main function to fetch tweets and parse them. \n",
    "        '''\n",
    "        # empty list to store parsed tweets \n",
    "        tweets = [] \n",
    "\n",
    "        try: \n",
    "            # call twitter api to fetch tweets \n",
    "            file_name = open(\"test.txt\",\"r\") \n",
    "            train = \"train.csv\"\n",
    "            csvf = open(\"train.csv\",\"w\")\n",
    "            csvwriter = csv.writer(csvf)\n",
    "            row = [\"text\",\"sentiment\"]\n",
    "            csvwriter.writerow(row)\n",
    "            for line in file_name:\n",
    "                fetched_tweets = self.api.search(q = line, count = count) \n",
    "  \n",
    "                # parsing tweets one by one \n",
    "                for tweet in fetched_tweets: \n",
    "                    # empty dictionary to store required params of a tweet \n",
    "                    parsed_tweet = {} \n",
    "  \n",
    "                    # saving text of tweet \n",
    "                    parsed_tweet['text'] = tweet.text \n",
    "                    # saving sentiment of tweet \n",
    "                    parsed_tweet['sentiment'] = self.get_tweet_sentiment(tweet.text) \n",
    "\n",
    "                    # appending parsed tweet to tweets list \n",
    "                    if tweet.retweet_count > 0: \n",
    "                        # if tweet has retweets, ensure that it is appended only once \n",
    "                        if parsed_tweet not in tweets: \n",
    "                            tweets.append(parsed_tweet)\n",
    "                            \n",
    "                    else: \n",
    "                        tweets.append(parsed_tweet)\n",
    "                    #tweets is our list \n",
    "                    f=0\n",
    "                    val1 = \"\"\n",
    "                    val2 = \"\"\n",
    "                    for i in tweets:\n",
    "                        for key, value in i.items():\n",
    "                            if(f==0):\n",
    "                                val1=value\n",
    "                                f=1\n",
    "                            else :\n",
    "                                f=0\n",
    "                                val2=value\n",
    "                        val1 = val1.replace(',', '')\n",
    "                        val3 = ' '.join([word for word in val1.split() if word not in  stopwords.words(\"english\")])\n",
    "                       # stops = set(stopwords.words('english'))\n",
    "                        row = [val3.encode(\"utf8\"),val2]\n",
    "                        csvwriter.writerow(row)  \n",
    "            #csvf.close()\n",
    "            return tweets \n",
    "        except tweepy.TweepError as e: \n",
    "            # print error (if any) \n",
    "            print(\"Error : \" + str(e)) \n",
    "\n",
    "def main(): \n",
    "    # creating object of TwitterClient Class \n",
    "    api = TwitterClient() \n",
    "    # calling function to get tweets \n",
    "    tweets = api.get_tweets(query = 'Donald Trump', count = 200) \n",
    "\n",
    "    # picking positive tweets from tweets \n",
    "    ptweets = [tweet for tweet in tweets if tweet['sentiment'] == 1] \n",
    "    # percentage of positive tweets \n",
    "    print(\"Positive tweets percentage: {} %\".format(100*len(ptweets)/len(tweets))) \n",
    "    # picking negative tweets from tweets \n",
    "    ntweets = [tweet for tweet in tweets if tweet['sentiment'] == -1] \n",
    "    # percentage of negative tweets \n",
    "    print(\"Negative tweets percentage: {} %\".format(100*len(ntweets)/len(tweets))) \n",
    "    # percentage of neutral tweets \n",
    "    #print(\"Neutral tweets percentage: {} %\".format(100*len(tweets - ntweets - ptweets)/len(tweets))) \n",
    "\n",
    "    # printing first 5 positive tweets \n",
    "    print(\"\\n\\nPositive tweets:\") \n",
    "    for tweet in ptweets[:10]: \n",
    "        print(tweet['text']) \n",
    "\n",
    "    # printing first 5 negative tweets \n",
    "    print(\"\\n\\nNegative tweets:\") \n",
    "    for tweet in ntweets[:10]: \n",
    "        print(tweet['text']) \n",
    "\n",
    "if __name__ == \"__main__\": \n",
    "    # calling main function \n",
    "    main() \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b'@tfwrail 2/2 managerial team ran ariva train...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b'@tfwrail 2/2 managerial team ran ariva train...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b'@tfwrail 2/2 managerial team ran ariva train...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>b'RT @cosmexbox: \\xe0\\xb8\\x9e\\xe0\\xb8\\xa3\\xe0\\...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>b'@tfwrail 2/2 managerial team ran ariva train...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>b'RT @cosmexbox: \\xe0\\xb8\\x9e\\xe0\\xb8\\xa3\\xe0\\...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>b'Shrewsbury accident emergency department ful...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>b'@tfwrail 2/2 managerial team ran ariva train...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>b'RT @cosmexbox: \\xe0\\xb8\\x9e\\xe0\\xb8\\xa3\\xe0\\...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>b'Shrewsbury accident emergency department ful...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>b'In news I tried playing Rivals last night I ...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>b'@tfwrail 2/2 managerial team ran ariva train...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>b'RT @cosmexbox: \\xe0\\xb8\\x9e\\xe0\\xb8\\xa3\\xe0\\...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>b'Shrewsbury accident emergency department ful...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>b'In news I tried playing Rivals last night I ...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>b\"@djcarlile @JPFinlayNBCS The defense looks r...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>b'@tfwrail 2/2 managerial team ran ariva train...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>b'RT @cosmexbox: \\xe0\\xb8\\x9e\\xe0\\xb8\\xa3\\xe0\\...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>b'Shrewsbury accident emergency department ful...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>b'In news I tried playing Rivals last night I ...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>b\"@djcarlile @JPFinlayNBCS The defense looks r...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>b'RT @V_Min_Hoseok: @smeraldusseesaw @TATA_adv...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>b'@tfwrail 2/2 managerial team ran ariva train...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143303</th>\n",
       "      <td>b'I abominate Rage Boiling deep inside To caug...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143304</th>\n",
       "      <td>b'@WhiteHouse @POTUS @realDonaldTrump Minds Ha...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143305</th>\n",
       "      <td>b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143306</th>\n",
       "      <td>b'In next days. Red? Why color sweet apples st...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143307</th>\n",
       "      <td>b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143308</th>\n",
       "      <td>b'@MarkGoulston Mortal beings fear understand....</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143309</th>\n",
       "      <td>b\"RT @sufjan_bot: Denominator go Decatur go De...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143310</th>\n",
       "      <td>b\"Denominator go Decatur go Decatur It's great...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143311</th>\n",
       "      <td>b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143312</th>\n",
       "      <td>b'\\xd0\\x98\\xd0\\xbd\\xd1\\x82\\xd0\\xb5\\xd1\\x80\\xd0...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143313</th>\n",
       "      <td>b'I HAVE every right criticize abominate thwar...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143314</th>\n",
       "      <td>b'I abominate Rage Boiling deep murder Enterin...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143315</th>\n",
       "      <td>b'abominate -- \\xed\\x98\\x90\\xec\\x98\\xa4\\xed\\x9...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143316</th>\n",
       "      <td>b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143317</th>\n",
       "      <td>b'abominate -- \\xed\\x98\\x90\\xec\\x98\\xa4\\xed\\x9...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143318</th>\n",
       "      <td>b\"@benjamindcrosby @nosdnomde All irenic theol...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143319</th>\n",
       "      <td>b'RT @EricHofferDaily: Ours golden age minorit...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143320</th>\n",
       "      <td>b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143321</th>\n",
       "      <td>b'Loathe | Definition Merriam-Webster hate det...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143322</th>\n",
       "      <td>b'abominate -- \\xed\\x98\\x90\\xec\\x98\\xa4\\xed\\x9...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143323</th>\n",
       "      <td>b\"I abominate Rage Boiling deep inside To caug...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143324</th>\n",
       "      <td>b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143325</th>\n",
       "      <td>b'Rev Dr Moszell MORITZ JAY BLACKMONAHAN WILL ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143326</th>\n",
       "      <td>b'RT @DoyleAbominator: Repost doylewolfgangvon...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143327</th>\n",
       "      <td>b'Repost @doylewolfgangvonfrankenstein using @...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143328</th>\n",
       "      <td>b'Repost doylewolfgangvonfrankenstein using re...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143329</th>\n",
       "      <td>b'Repost doylewolfgangvonfrankenstein using re...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143330</th>\n",
       "      <td>b'\\xe7\\xa7\\x81\\xe3\\x80\\x81\\xe5\\xad\\xa6\\xe5\\x8b...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143331</th>\n",
       "      <td>b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143332</th>\n",
       "      <td>b'Rev Dr Moszell MORITZ JAY BLACKMONAHAN WILL ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>143333 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  sentiment\n",
       "0       b'@tfwrail 2/2 managerial team ran ariva train...         -1\n",
       "1       b'@tfwrail 2/2 managerial team ran ariva train...         -1\n",
       "2       b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...          0\n",
       "3       b'@tfwrail 2/2 managerial team ran ariva train...         -1\n",
       "4       b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...          0\n",
       "5       b'RT @cosmexbox: \\xe0\\xb8\\x9e\\xe0\\xb8\\xa3\\xe0\\...          0\n",
       "6       b'@tfwrail 2/2 managerial team ran ariva train...         -1\n",
       "7       b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...          0\n",
       "8       b'RT @cosmexbox: \\xe0\\xb8\\x9e\\xe0\\xb8\\xa3\\xe0\\...          0\n",
       "9       b'Shrewsbury accident emergency department ful...         -1\n",
       "10      b'@tfwrail 2/2 managerial team ran ariva train...         -1\n",
       "11      b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...          0\n",
       "12      b'RT @cosmexbox: \\xe0\\xb8\\x9e\\xe0\\xb8\\xa3\\xe0\\...          0\n",
       "13      b'Shrewsbury accident emergency department ful...         -1\n",
       "14      b'In news I tried playing Rivals last night I ...         -1\n",
       "15      b'@tfwrail 2/2 managerial team ran ariva train...         -1\n",
       "16      b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...          0\n",
       "17      b'RT @cosmexbox: \\xe0\\xb8\\x9e\\xe0\\xb8\\xa3\\xe0\\...          0\n",
       "18      b'Shrewsbury accident emergency department ful...         -1\n",
       "19      b'In news I tried playing Rivals last night I ...         -1\n",
       "20      b\"@djcarlile @JPFinlayNBCS The defense looks r...          1\n",
       "21      b'@tfwrail 2/2 managerial team ran ariva train...         -1\n",
       "22      b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...          0\n",
       "23      b'RT @cosmexbox: \\xe0\\xb8\\x9e\\xe0\\xb8\\xa3\\xe0\\...          0\n",
       "24      b'Shrewsbury accident emergency department ful...         -1\n",
       "25      b'In news I tried playing Rivals last night I ...         -1\n",
       "26      b\"@djcarlile @JPFinlayNBCS The defense looks r...          1\n",
       "27      b'RT @V_Min_Hoseok: @smeraldusseesaw @TATA_adv...          0\n",
       "28      b'@tfwrail 2/2 managerial team ran ariva train...         -1\n",
       "29      b'RT @aruna_sekhar: @IndEditorsGuild Notes sta...          0\n",
       "...                                                   ...        ...\n",
       "143303  b'I abominate Rage Boiling deep inside To caug...         -1\n",
       "143304  b'@WhiteHouse @POTUS @realDonaldTrump Minds Ha...         -1\n",
       "143305  b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...          0\n",
       "143306  b'In next days. Red? Why color sweet apples st...          1\n",
       "143307  b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...          0\n",
       "143308  b'@MarkGoulston Mortal beings fear understand....          0\n",
       "143309  b\"RT @sufjan_bot: Denominator go Decatur go De...          1\n",
       "143310  b\"Denominator go Decatur go Decatur It's great...          0\n",
       "143311  b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...          0\n",
       "143312  b'\\xd0\\x98\\xd0\\xbd\\xd1\\x82\\xd0\\xb5\\xd1\\x80\\xd0...         -1\n",
       "143313  b'I HAVE every right criticize abominate thwar...          1\n",
       "143314  b'I abominate Rage Boiling deep murder Enterin...          1\n",
       "143315  b'abominate -- \\xed\\x98\\x90\\xec\\x98\\xa4\\xed\\x9...          0\n",
       "143316  b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...          0\n",
       "143317  b'abominate -- \\xed\\x98\\x90\\xec\\x98\\xa4\\xed\\x9...          0\n",
       "143318  b\"@benjamindcrosby @nosdnomde All irenic theol...          0\n",
       "143319  b'RT @EricHofferDaily: Ours golden age minorit...          1\n",
       "143320  b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...          0\n",
       "143321  b'Loathe | Definition Merriam-Webster hate det...         -1\n",
       "143322  b'abominate -- \\xed\\x98\\x90\\xec\\x98\\xa4\\xed\\x9...          0\n",
       "143323  b\"I abominate Rage Boiling deep inside To caug...         -1\n",
       "143324  b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...          0\n",
       "143325  b'Rev Dr Moszell MORITZ JAY BLACKMONAHAN WILL ...          1\n",
       "143326  b'RT @DoyleAbominator: Repost doylewolfgangvon...          0\n",
       "143327  b'Repost @doylewolfgangvonfrankenstein using @...          0\n",
       "143328  b'Repost doylewolfgangvonfrankenstein using re...          0\n",
       "143329  b'Repost doylewolfgangvonfrankenstein using re...          0\n",
       "143330  b'\\xe7\\xa7\\x81\\xe3\\x80\\x81\\xe5\\xad\\xa6\\xe5\\x8b...          0\n",
       "143331  b'Rev. Dr. Moszell MORITZ JAY BLACKMONAHAN WIL...          0\n",
       "143332  b'Rev Dr Moszell MORITZ JAY BLACKMONAHAN WILL ...          1\n",
       "\n",
       "[143333 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df=pd.read_csv(\"train.csv\");\n",
    "df1 = df\n",
    "df1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets, linear_model\n",
    "from sklearn.model_selection import train_test_split\n",
    "from matplotlib import pyplot as plt\n",
    "y = df['sentiment']\n",
    "x = df['text']\n",
    "X_train, X_test, y_train, y_test = train_test_split(df['text'],df['sentiment'] , test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_PROCESSED_FILE = 'train-processed.csv'\n",
    "TEST_PROCESSED_FILE = 'test-processed.csv'\n",
    "TEST_PROCESSED_FILE_CHECK = 'test-processed-check.csv'\n",
    "X_test.to_csv(TEST_PROCESSED_FILE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_train = pd.concat([X_train, y_train,], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_train.to_csv(TRAIN_PROCESSED_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import utils\n",
    "\n",
    "# Classifies a tweet based on the number of positive and negative words in it\n",
    "POSITIVE_WORDS_FILE = 'positive-words.txt'\n",
    "NEGATIVE_WORDS_FILE = 'negative-words.txt'\n",
    "TRAIN = True\n",
    "\n",
    "def funct(l):\n",
    "    if(l==\"negative\"):\n",
    "        return -1\n",
    "    elif(l==\"positive\"):\n",
    "        return 1\n",
    "    else :\n",
    "        return 0\n",
    "def classify(processed_csv, test_file=True, **params):\n",
    "    positive_words = utils.file_to_wordset(params.pop('positive_words'))\n",
    "    negative_words = utils.file_to_wordset(params.pop('negative_words'))\n",
    "    predictions = []\n",
    "    with open(processed_csv, 'r') as csv:\n",
    "        for line in csv:\n",
    "            if test_file:\n",
    "                tweet_id, tweet = line.strip().split(',')\n",
    "            else:\n",
    "                tweet_id,tweet,label= line.strip().split(',')\n",
    "            pos_count, neg_count = 0, 0\n",
    "            for word in tweet.split():\n",
    "                if word in positive_words:\n",
    "                    pos_count += 1\n",
    "                elif word in negative_words:\n",
    "                    neg_count += 1\n",
    "            # print pos_count, neg_count\n",
    "            prediction = 1 if pos_count >= neg_count else 0\n",
    "            if test_file:\n",
    "                predictions.append((tweet_id, prediction))\n",
    "            else:\n",
    "                predictions.append((tweet_id, funct(label), prediction))\n",
    "    return predictions\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    if TRAIN:\n",
    "        predictions = classify(TRAIN_PROCESSED_FILE, test_file=(not TRAIN), positive_words=POSITIVE_WORDS_FILE, negative_words=NEGATIVE_WORDS_FILE)\n",
    "        correct = sum([1 for p in predictions if p[1] == p[2]]) * 100.0 / len(predictions)\n",
    "        print('Correct = %.2f%%' % correct)\n",
    "    else:\n",
    "        predictions = classify(TEST_PROCESSED_FILE, test_file=(not TRAIN), positive_words=POSITIVE_WORDS_FILE, negative_words=NEGATIVE_WORDS_FILE)\n",
    "        utils.save_results_to_csv(predictions, 'baseline.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FREQ_DIST_FILE = \"twitter-sentiment-analysis/code/train-processed-freqdist.pkl\"\n",
    "unigrams = utils.top_n_words(FREQ_DIST_FILE, 100)\n",
    "FREQ_DIST_FILE_bi = \"twitter-sentiment-analysis/code/train-processed-freqdist-bi.pkl\"\n",
    "bigrams = utils.top_n_words(FREQ_DIST_FILE_bi, 100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_BIGRAMS = True\n",
    "def get_feature_vector(tweet):\n",
    "    uni_feature_vector = []\n",
    "    bi_feature_vector = []\n",
    "    words = tweet.split()\n",
    "    for i in range(len(words) - 1):\n",
    "        word = words[i]\n",
    "        next_word = words[i + 1]\n",
    "        if unigrams.get(word):\n",
    "            uni_feature_vector.append(word)\n",
    "        if USE_BIGRAMS:\n",
    "            if bigrams.get((word, next_word)):\n",
    "                bi_feature_vector.append((word, next_word))\n",
    "    if len(words) >= 1:\n",
    "        if unigrams.get(words[-1]):\n",
    "            uni_feature_vector.append(words[-1])\n",
    "    return uni_feature_vector, bi_feature_vector\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def process_tweets(csv_file, test_file=True):\n",
    "    \"\"\"Returns a list of tuples of type (tweet_id, feature_vector)\n",
    "            or (tweet_id, sentiment, feature_vector)\n",
    "\n",
    "    Args:\n",
    "        csv_file (str): Name of processed csv file generated by preprocess.py\n",
    "        test_file (bool, optional): If processing test file\n",
    "\n",
    "    Returns:\n",
    "        list: Of tuples\n",
    "    \"\"\"\n",
    "    tweets = []\n",
    "    print('Generating feature vectors')\n",
    "    with open(csv_file, 'r') as csv:\n",
    "        lines = csv.readlines()\n",
    "        total = len(lines)\n",
    "        for i, line in enumerate(lines):\n",
    "            if test_file:\n",
    "                tweet_id, tweet = line.split(',')\n",
    "            else:\n",
    "                tweet_id,  tweet, sentiment = line.split(',')\n",
    "            feature_vector = get_feature_vector(tweet)\n",
    "            if test_file:\n",
    "                tweets.append((tweet_id, feature_vector))\n",
    "            else:\n",
    "                tweets.append((tweet_id, sentiment, feature_vector))\n",
    "            utils.write_status(i + 1, total)\n",
    "    print('\\n')\n",
    "    return tweets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tweets = process_tweets(TEST_PROCESSED_FILE, test_file=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tweets = process_tweets(TRAIN_PROCESSED_FILE, test_file=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "for i,tweet in enumerate(train_tweets):\n",
    "    if i!=0:\n",
    "        f = 1.0\n",
    "        f = float(tweet[1])\n",
    "        print(f)\n",
    "    #print(ast.literal_eval(tweet[1]))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,tweet in enumerate(test_tweets):\n",
    "    print(tweet[1][1])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "with open(TRAIN_PROCESSED_FILE, 'r') as csv:\n",
    "        lines = csv.readlines()\n",
    "        #print(lines)\n",
    "        total = len(lines)\n",
    "        for i, line in enumerate(lines):\n",
    "            line = line.strip().split(',')\n",
    "            print(i)\n",
    "            print(line[2])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}