preparing dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "787d1b81",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import sys\n",
    "from utils import write_status\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk import FreqDist\n",
    "import pickle\n",
    "import sys\n",
    "from utils import write_status\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bf8b740e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_word(word):\n",
    "    word = word.strip('\\'\"?!,.():;')\n",
    "    word = re.sub(r'(.)\\1+', r'\\1\\1', word)\n",
    "    word = re.sub(r'(-|\\')', '', word)\n",
    "    return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4816b28e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_valid_word(word):\n",
    "    # Check if word begins with an alphabet\n",
    "    return (re.search(r'^[a-zA-Z][a-z0-9A-Z\\._]*$', word) is not None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "174c0bbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "urlfil=\"F:/dataset/US airline-data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3eec5d99",
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_emojis(tweet):\n",
    "    # Smile -- :), : ), :-), (:, ( :, (-:, :')\n",
    "    tweet = re.sub(r'(:\\s?\\)|:-\\)|\\(\\s?:|\\(-:|:\\'\\))', ' EMO_POS ', tweet)\n",
    "    # Laugh -- :D, : D, :-D, xD, x-D, XD, X-D\n",
    "    tweet = re.sub(r'(:\\s?D|:-D|x-?D|X-?D)', ' EMO_POS ', tweet)\n",
    "    # Love -- <3, :*\n",
    "    tweet = re.sub(r'(<3|:\\*)', ' EMO_POS ', tweet)\n",
    "    # Wink -- ;-), ;), ;-D, ;D, (;,  (-;\n",
    "    tweet = re.sub(r'(;-?\\)|;-?D|\\(-?;)', ' EMO_POS ', tweet)\n",
    "    # Sad -- :-(, : (, :(, ):, )-:\n",
    "    tweet = re.sub(r'(:\\s?\\(|:-\\(|\\)\\s?:|\\)-:)', ' EMO_NEG ', tweet)\n",
    "    # Cry -- :,(, :'(, :\"(\n",
    "    tweet = re.sub(r'(:,\\(|:\\'\\(|:\"\\()', ' EMO_NEG ', tweet)\n",
    "    return tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0f191e29",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_tweet(tweet):\n",
    "    processed_tweet = []\n",
    "    # Convert to lower case\n",
    "    tweet = tweet.lower()\n",
    "    # Replaces URLs with the word URL\n",
    "    tweet = re.sub(r'((www\\.[\\S]+)|(https?://[\\S]+))', ' URL ', tweet)\n",
    "    # Replace @handle with the word USER_MENTION\n",
    "    tweet = re.sub(r'@[\\S]+', 'USER_MENTION', tweet)\n",
    "    # Replaces #hashtag with hashtag\n",
    "    tweet = re.sub(r'#(\\S+)', r' \\1 ', tweet)\n",
    "    # Remove RT (retweet)\n",
    "    tweet = re.sub(r'\\brt\\b', '', tweet)\n",
    "    # Replace 2+ dots with space\n",
    "    tweet = re.sub(r'\\.{2,}', ' ', tweet)\n",
    "    # Strip space, \" and ' from tweet\n",
    "    tweet = tweet.strip(' \"\\'')\n",
    "    # Replace emojis with either EMO_POS or EMO_NEG\n",
    "    tweet = handle_emojis(tweet)\n",
    "    # Replace multiple spaces with a single space\n",
    "    tweet = re.sub(r'\\s+', ' ', tweet)\n",
    "    words = tweet.split()\n",
    "\n",
    "    for word in words:\n",
    "        word = preprocess_word(word)\n",
    "        if is_valid_word(word):\n",
    "            if use_stemmer:\n",
    "                word = str(porter_stemmer.stem(word))\n",
    "            processed_tweet.append(word)\n",
    "\n",
    "    return ' '.join(processed_tweet)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9b0da570",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_csv(csv_file_name, processed_file_name, test_file=False):\n",
    "    save_to_file = open(processed_file_name, 'w')\n",
    "    with open(csv_file_name, 'r', encoding = \"ISO-8859-1\") as csv:\n",
    "        lines = csv.readlines()\n",
    "        total = len(lines)\n",
    "        for i, line in enumerate(lines):\n",
    "            tweet_id = line[:line.find(',')]\n",
    "            if not test_file:\n",
    "                line = line[1 + line.find(','):]\n",
    "                positive = int(line[:line.find(',')])\n",
    "            line = line[1 + line.find(','):]\n",
    "            tweet = line\n",
    "            processed_tweet = preprocess_tweet(tweet)\n",
    "            if not test_file:\n",
    "                save_to_file.write('%s,%d,%s\\n' %\n",
    "                                   (tweet_id, positive, processed_tweet))\n",
    "            else:\n",
    "                save_to_file.write('%s,%s\\n' %\n",
    "                                   (tweet_id, processed_tweet))\n",
    "            write_status(i + 1, total)\n",
    "    save_to_file.close()\n",
    "    print ('\\nSaved processed tweets to: %s' % processed_file_name)\n",
    "    return processed_file_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b2d2ca88",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kamyab\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py\n",
      "F:/dataset/US airline-data/processed_data.csv\n",
      "Processing 2978/14640"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing 7449/14640"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing 11982/14640"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing 14640/14640\n",
      "Saved processed tweets to: F:/dataset/US airline-data/processed_data.csv\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'F:/dataset/US airline-data/processed_data.csv'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if len(sys.argv) != 2:\n",
    "    print(sys.argv[0])\n",
    "use_stemmer = False\n",
    "csv_file_name = urlfil+\"dataset.csv\"\n",
    "processed_file_name = urlfil + 'processed_data.csv'\n",
    "print(processed_file_name)\n",
    "preprocess_csv(csv_file_name, processed_file_name, test_file=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1bb06efc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze_tweet(tweet):\n",
    "    result = {}\n",
    "    result['MENTIONS'] = tweet.count('USER_MENTION')\n",
    "    result['URLS'] = tweet.count('URL')\n",
    "    result['POS_EMOS'] = tweet.count('EMO_POS')\n",
    "    result['NEG_EMOS'] = tweet.count('EMO_NEG')\n",
    "    tweet = tweet.replace('USER_MENTION', '').replace(\n",
    "        'URL', '')\n",
    "    words = tweet.split()\n",
    "    result['WORDS'] = len(words)\n",
    "    bigrams = get_bigrams(words)\n",
    "    result['BIGRAMS'] = len(bigrams)\n",
    "    return result, words, bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a3e539c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bigrams(tweet_words):\n",
    "    bigrams = []\n",
    "    num_words = len(tweet_words)\n",
    "    for i in range(num_words - 1):\n",
    "        bigrams.append((tweet_words[i], tweet_words[i + 1]))\n",
    "    return bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cee97ff1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bigram_freqdist(bigrams):\n",
    "    freq_dict = {}\n",
    "    for bigram in bigrams:\n",
    "        if freq_dict.get(bigram):\n",
    "            freq_dict[bigram] += 1\n",
    "        else:\n",
    "            freq_dict[bigram] = 1\n",
    "    counter = Counter(freq_dict)\n",
    "    return counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "53ff02dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing 2972/14640"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing 7512/14640"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing 12087/14640"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing 14640/14640\n",
      "Calculating frequency distribution\n",
      "Saved uni-frequency distribution to F:/dataset/US airline-data/processed-freqdist.pkl\n",
      "Saved bi-frequency distribution to F:/dataset/US airline-data/processed-freqdist-bi.pkl\n",
      "\n",
      "[Analysis Statistics]\n",
      "Tweets => Total: 14640, Positive: 11541, Negative: 3099\n",
      "User Mentions => Total: 16318, Avg: 1.1146, Max: 6\n",
      "URLs => Total: 1211, Avg: 0.0827, Max: 3\n",
      "Emojis => Total: 343, Positive: 241, Negative: 102, Avg: 0.0234, Max: 2\n",
      "Words => Total: 230725, Unique: 11977, Avg: 15.7599, Max: 30, Min: 0\n",
      "Bigrams => Total: 216090, Unique: 89348, Avg: 14.7602\n"
     ]
    }
   ],
   "source": [
    "num_tweets, num_pos_tweets, num_neg_tweets = 0, 0, 0\n",
    "num_mentions, max_mentions = 0, 0\n",
    "num_emojis, num_pos_emojis, num_neg_emojis, max_emojis = 0, 0, 0, 0\n",
    "num_urls, max_urls = 0, 0\n",
    "num_words, num_unique_words, min_words, max_words = 0, 0, 1e6, 0\n",
    "num_bigrams, num_unique_bigrams = 0, 0\n",
    "all_words = []\n",
    "all_bigrams = []\n",
    "with open(urlfil+'processed_data.csv', 'r') as csv:\n",
    "    lines = csv.readlines()\n",
    "    num_tweets = len(lines)\n",
    "    for i, line in enumerate(lines):\n",
    "        t_id, if_pos, tweet = line.strip().split(',')\n",
    "        if_pos = int(if_pos)\n",
    "        if if_pos:\n",
    "            num_pos_tweets += 1\n",
    "        else:\n",
    "            num_neg_tweets += 1\n",
    "        result, words, bigrams = analyze_tweet(tweet)\n",
    "        num_mentions += result['MENTIONS']\n",
    "        max_mentions = max(max_mentions, result['MENTIONS'])\n",
    "        num_pos_emojis += result['POS_EMOS']\n",
    "        num_neg_emojis += result['NEG_EMOS']\n",
    "        max_emojis = max(\n",
    "            max_emojis, result['POS_EMOS'] + result['NEG_EMOS'])\n",
    "        num_urls += result['URLS']\n",
    "        max_urls = max(max_urls, result['URLS'])\n",
    "        num_words += result['WORDS']\n",
    "        min_words = min(min_words, result['WORDS'])\n",
    "        max_words = max(max_words, result['WORDS'])\n",
    "        all_words.extend(words)\n",
    "        num_bigrams += result['BIGRAMS']\n",
    "        all_bigrams.extend(bigrams)\n",
    "        write_status(i + 1, num_tweets)\n",
    "num_emojis = num_pos_emojis + num_neg_emojis\n",
    "unique_words = list(set(all_words))\n",
    "with open(urlfil+'processed-unique.txt', 'w') as uwf:\n",
    "    uwf.write('\\n'.join(unique_words))\n",
    "num_unique_words = len(unique_words)\n",
    "num_unique_bigrams = len(set(all_bigrams))\n",
    "print ('\\nCalculating frequency distribution')\n",
    "# Unigrams\n",
    "freq_dist = FreqDist(all_words)\n",
    "pkl_file_name = urlfil+'processed-freqdist.pkl'\n",
    "with open(pkl_file_name, 'wb') as pkl_file:\n",
    "    pickle.dump(freq_dist, pkl_file)\n",
    "print ('Saved uni-frequency distribution to %s' % pkl_file_name)\n",
    "# Bigrams\n",
    "bigram_freq_dist = get_bigram_freqdist(all_bigrams)\n",
    "bi_pkl_file_name = urlfil+'processed-freqdist-bi.pkl'\n",
    "with open(bi_pkl_file_name, 'wb') as pkl_file:\n",
    "    pickle.dump(bigram_freq_dist, pkl_file)\n",
    "print ('Saved bi-frequency distribution to %s' % bi_pkl_file_name)\n",
    "print ('\\n[Analysis Statistics]')\n",
    "print ('Tweets => Total: %d, Positive: %d, Negative: %d' % (num_tweets, num_pos_tweets, num_neg_tweets))\n",
    "print ('User Mentions => Total: %d, Avg: %.4f, Max: %d' % (num_mentions, num_mentions / float(num_tweets), max_mentions))\n",
    "print ('URLs => Total: %d, Avg: %.4f, Max: %d' % (num_urls, num_urls / float(num_tweets), max_urls))\n",
    "print ('Emojis => Total: %d, Positive: %d, Negative: %d, Avg: %.4f, Max: %d' % (num_emojis, num_pos_emojis, num_neg_emojis, num_emojis / float(num_tweets), max_emojis))\n",
    "print ('Words => Total: %d, Unique: %d, Avg: %.4f, Max: %d, Min: %d' % (num_words, num_unique_words, num_words / float(num_tweets), max_words, min_words))\n",
    "print ('Bigrams => Total: %d, Unique: %d, Avg: %.4f' % (num_bigrams, num_unique_bigrams, num_bigrams / float(num_tweets)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "190209b4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}