featurization/VectorizerBuilder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import string\n",
    "import gzip\n",
    "import json\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy\n",
    "from scipy import interpolate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/.local/lib/python3.6/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    }
   ],
   "source": [
    "import sklearn.cluster\n",
    "import sklearn.feature_extraction \n",
    "import sklearn.feature_extraction.text\n",
    "import sklearn.metrics\n",
    "import sklearn.preprocessing\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.semi_supervised import LabelPropagation\n",
    "from sklearn.semi_supervised import LabelSpreading\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/clb617/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/.local/lib/python3.6/site-packages/nltk/twitter/__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
      "  warnings.warn(\"The twython library has not been installed. \"\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download(\"stopwords\")\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer as VS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# But first, read in stopwrods\n",
    "enStop = stopwords.words('english')\n",
    "\n",
    "# Skip stop words, retweet signs, @ symbols, and URL headers\n",
    "stopList = enStop +\\\n",
    "    [\"http\", \"https\", \"rt\", \"@\", \":\", \"t.co\", \"co\", \"amp\", \"&amp;\", \"...\", \"\\n\", \"\\r\"]\n",
    "stopList.extend(string.punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def tokenizer_wrapper(text):\n",
    "#     return [t.lemma_ for t in nlp(text)]\n",
    "\n",
    "local_tokenizer = TweetTokenizer()\n",
    "def tokenizer_wrapper(text):\n",
    "    return local_tokenizer.tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectorizer = sklearn.feature_extraction.text.TfidfVectorizer(\n",
    "    tokenizer=tokenizer_wrapper,\n",
    "    ngram_range=(1, 2),\n",
    "    stop_words=stopList, #We do better when we keep stopwords\n",
    "    use_idf=True,\n",
    "    smooth_idf=False,\n",
    "    norm=None, #Applies l2 norm smoothing\n",
    "    decode_error='replace',\n",
    "    max_features=10000,\n",
    "    min_df=4,\n",
    "    max_df=0.501\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tweet_texts = []\n",
    "\n",
    "for f in glob.glob(\"tweet_random_subset_2013to2016_v2/part*.gz\"):\n",
    "    flag = False\n",
    "    with gzip.open(f, \"r\") as in_file:\n",
    "        for line_bytes in in_file:\n",
    "            line = line_bytes.decode(\"utf8\")\n",
    "\n",
    "            if ( len(line.strip()) == 0 ):\n",
    "                continue\n",
    "                \n",
    "            tweet = json.loads(line)\n",
    "            text = tweet[\"text\"]\n",
    "\n",
    "            tweet_texts.append(text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tweet Count: 11715393\n"
     ]
    }
   ],
   "source": [
    "print(\"Tweet Count:\", len(tweet_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trec_tweets = []\n",
    "with open(\"rehydrated_tweets.json\", \"r\") as in_file:\n",
    "    for line in in_file:\n",
    "        tweet = json.loads(line)\n",
    "        trec_tweets.append(tweet[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_texts = trec_tweets + tweet_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer.fit(all_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joblib.dump(vectorizer, \"2013to2016_tfidf_vectorizer_20190109.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = {v:i for i, v in enumerate(vectorizer.get_feature_names())}\n",
    "idf_vals = vectorizer.idf_\n",
    "idf_dict = {i:idf_vals[i] for i in vocab.values()} #keys are indices; values are IDF scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary Word Count: 10000\n",
      "['🚨', '🚨 🚨', '🚫', '🚮', '🚶', '🤑', '🤔', '🤔 🤔', '🤗', '🤘']\n"
     ]
    }
   ],
   "source": [
    "print(\"Dictionary Word Count:\", len(vocab))\n",
    "print([x[0] for x in vocab.items()][-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished\n"
     ]
    }
   ],
   "source": [
    "print(\"Finished\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}