N-gram Statistical Approach.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "from symspellpy.symspellpy import SymSpell, Verbosity\n",
    "import pkg_resources\n",
    "import pickle\n",
    "from num2words import num2words\n",
    "import re, string, json\n",
    "from tqdm.notebook import tqdm\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "nltk.data.path.append('.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simplify_punctuation_and_whitespace(sentence_list):\n",
    "    norm_sents = []\n",
    "    print(\"Normalizing whitespaces and punctuation\")\n",
    "    for sentence in tqdm(sentence_list):\n",
    "        sent = _replace_urls(sentence)\n",
    "        sent = _simplify_punctuation(sentence)\n",
    "        sent = _extra_chars(sentence)\n",
    "        sent = _normalize_whitespace(sent)\n",
    "        norm_sents.append(sent)\n",
    "    return norm_sents\n",
    "\n",
    "def _replace_urls(text):\n",
    "    url_regex = r'(https?:\\/\\/(?:www\\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\\.[^\\s]{2,}|www\\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\\.[^\\s]{2,}|https?:\\/\\/(?:www\\.|(?!www))[a-zA-Z0-9]+\\.[^\\s]{2,}|www\\.[a-zA-Z0-9]+\\.[^\\s]{2,})'\n",
    "    text = re.sub(url_regex, \"\", text)\n",
    "    return text\n",
    "\n",
    "def _simplify_punctuation(text):\n",
    "    \"\"\"\n",
    "    This function simplifies doubled or more complex punctuation. The exception is '...'.\n",
    "    \"\"\"\n",
    "    corrected = str(text)\n",
    "    corrected = re.sub(r'([!?,;])\\1+', r'\\1', corrected)\n",
    "    corrected = re.sub(r'\\.{2,}', r'...', corrected)\n",
    "    return corrected\n",
    "\n",
    "def _normalize_whitespace(text):\n",
    "    \"\"\"\n",
    "    This function normalizes whitespaces, removing duplicates.\n",
    "    \"\"\"\n",
    "    corrected = str(text)\n",
    "    corrected = re.sub(r\"//t\",r\"\\t\", corrected)\n",
    "    corrected = re.sub(r\"( )\\1+\",r\"\\1\", corrected)\n",
    "    corrected = re.sub(r\"(\\n)\\1+\",r\"\\1\", corrected)\n",
    "    corrected = re.sub(r\"(\\r)\\1+\",r\"\\1\", corrected)\n",
    "    corrected = re.sub(r\"(\\t)\\1+\",r\"\\1\", corrected)\n",
    "    return corrected.strip(\" \")\n",
    "\n",
    "def _extra_chars(text):\n",
    "    new = re.sub(r\"[^a-zA-Z0-9]+\", ' ', text)# as a last resort i guess.\n",
    "    return new\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_contractions(sentence_list):\n",
    "    contraction_list = json.loads(open('english_contractions.json', 'r').read())\n",
    "    norm_sents = []\n",
    "    print(\"Normalizing contractions\")\n",
    "    for sentence in tqdm(sentence_list):\n",
    "        norm_sents.append(_normalize_contractions_text(sentence, contraction_list))\n",
    "    return norm_sents\n",
    "\n",
    "def _normalize_contractions_text(text, contractions):\n",
    "    \"\"\"\n",
    "    This function normalizes english contractions.\n",
    "    \"\"\"\n",
    "    new_token_list = []\n",
    "    token_list = text.split()\n",
    "    for word_pos in range(len(token_list)):\n",
    "        word = token_list[word_pos]\n",
    "        first_upper = False\n",
    "        if word[0].isupper():\n",
    "            first_upper = True\n",
    "        if word.lower() in contractions:\n",
    "            replacement = contractions[word.lower()]\n",
    "            if first_upper:\n",
    "                replacement = replacement[0].upper()+replacement[1:]\n",
    "            replacement_tokens = replacement.split()\n",
    "            if len(replacement_tokens)>1:\n",
    "                new_token_list.append(replacement_tokens[0])\n",
    "                new_token_list.append(replacement_tokens[1])\n",
    "            else:\n",
    "                new_token_list.append(replacement_tokens[0])\n",
    "        else:\n",
    "            new_token_list.append(word)\n",
    "    sentence = \" \".join(new_token_list).strip(\" \")\n",
    "    return sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def spell_correction(sentence_list):\n",
    "    max_edit_distance_dictionary= 3\n",
    "    prefix_length = 4\n",
    "    spellchecker = SymSpell(max_edit_distance_dictionary, prefix_length)\n",
    "    dictionary_path = pkg_resources.resource_filename(\n",
    "        \"symspellpy\", \"frequency_dictionary_en_82_765.txt\")\n",
    "    bigram_path = pkg_resources.resource_filename(\n",
    "        \"symspellpy\", \"frequency_bigramdictionary_en_243_342.txt\")\n",
    "    spellchecker.load_dictionary(dictionary_path, term_index=0, count_index=1)\n",
    "    spellchecker.load_bigram_dictionary(dictionary_path, term_index=0, count_index=2)\n",
    "    norm_sents = []\n",
    "    print(\"Spell correcting\")\n",
    "    for sentence in tqdm(sentence_list):\n",
    "        norm_sents.append(_spell_correction_text(sentence, spellchecker))\n",
    "    return norm_sents\n",
    "\n",
    "def _spell_correction_text(text, spellchecker):\n",
    "    \"\"\"\n",
    "    This function does very simple spell correction normalization using pyspellchecker module. It works over a tokenized sentence and only the token representations are changed.\n",
    "    \"\"\"\n",
    "    if len(text) < 1:\n",
    "        return \"\"\n",
    "    #Spell checker config\n",
    "    max_edit_distance_lookup = 2\n",
    "    suggestion_verbosity = Verbosity.TOP # TOP, CLOSEST, ALL\n",
    "    #End of Spell checker config\n",
    "    token_list = text.split()\n",
    "    for word_pos in range(len(token_list)):\n",
    "        word = token_list[word_pos]\n",
    "        if word is None:\n",
    "            token_list[word_pos] = \"\"\n",
    "            continue\n",
    "        if not '\\n' in word and word not in string.punctuation and not is_numeric(word) and not (word.lower() in spellchecker.words.keys()):\n",
    "            suggestions = spellchecker.lookup(word.lower(), suggestion_verbosity, max_edit_distance_lookup)\n",
    "            #Checks first uppercase to conserve the case.\n",
    "            upperfirst = word[0].isupper()\n",
    "            #Checks for correction suggestions.\n",
    "            if len(suggestions) > 0:\n",
    "                correction = suggestions[0].term\n",
    "                replacement = correction\n",
    "            #We call our _reduce_exaggerations function if no suggestion is found. Maybe there are repeated chars.\n",
    "            else:\n",
    "                replacement = _reduce_exaggerations(word)\n",
    "            #Takes the case back to the word.\n",
    "            if upperfirst:\n",
    "                replacement = replacement[0].upper()+replacement[1:]\n",
    "            word = replacement\n",
    "            token_list[word_pos] = word\n",
    "    return \" \".join(token_list).strip()\n",
    "\n",
    "def _reduce_exaggerations(text):\n",
    "    \"\"\"\n",
    "    Auxiliary function to help with exxagerated words.\n",
    "    Examples:\n",
    "        woooooords -> words\n",
    "        yaaaaaaaaaaaaaaay -> yay\n",
    "    \"\"\"\n",
    "    correction = str(text)\n",
    "    #TODO work on complexity reduction.\n",
    "    return re.sub(r'([\\w])\\1+', r'\\1', correction)\n",
    "\n",
    "def is_numeric(text):\n",
    "    temp = []\n",
    "    for char in text:\n",
    "        if not (char in \"0123456789\" or char in \",%$\"):\n",
    "            return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Number_change(text):\n",
    "    temp = []\n",
    "    for char in text:\n",
    "        if not (char in \"0123456789\"):\n",
    "            temp.append(char)\n",
    "        elif (char in \"0123456789\" for i in range(len(char))):\n",
    "            temp.append(num2words(int(char), lang ='es'))\n",
    "    return temp\n",
    "\n",
    "def Tokeni(sentence):\n",
    "    tokenized = word_tokenize(sentence)\n",
    "    return tokenized\n",
    "\n",
    "def lemma(sentences):\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    final = []\n",
    "    for sentence in tqdm(sentences):\n",
    "        word_list = Tokeni(sentence)\n",
    "        word_list = Number_change(word_list)\n",
    "        lemmatized_output = [lemmatizer.lemmatize(w) for w in word_list]\n",
    "        final.append(lemmatized_output)\n",
    "    \n",
    "    return final"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalization_pipeline(sentences):\n",
    "    print(\"Starting Normalization Process\")\n",
    "    sentences = simplify_punctuation_and_whitespace(sentences)\n",
    "    sentences = normalize_contractions(sentences)\n",
    "    # sentences = spell_correction(sentences)\n",
    "    sentences = lemma(sentences)\n",
    "    print(\"Normalization Process Finished\")\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_to_sentences(data):\n",
    "    \"\"\"\n",
    "    Split data by linebreak \".\"\n",
    "\n",
    "    Args:\n",
    "        data: str\n",
    "\n",
    "    Returns:\n",
    "        A list of sentences\n",
    "    \"\"\"\n",
    "    sentences = data.split('.') # \n",
    "    sentences = [s.strip() for s in sentences]\n",
    "    sentences = [s.lower() for s in sentences if len(s) > 3] # \n",
    "\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For each type of data you have to configure the split to sentece function accordingly.\n",
    "# result comes in seconds\n",
    "# to_sentences = split_to_sentences(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As you can see even with splitting and checking length the data still hass noise in it.\n",
    "# For **twitter data** we would need all functions in our pipeline.\n",
    "# Though from the below results we can see that user names like rGXNogHMKM are still not removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final_data = normalization_pipeline(to_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pkl_file = open('DataSet/Pickle/Twitter_mix.pickle','wb')\n",
    "# pickle.dump(final_data, pkl_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['edu', 'eric', 'sieferman', 'subject', 're', 'some', 'thought']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('DataSet/Pickle/News Mixed.pickle', 'rb') as f:\n",
    "    data = pickle.load(f)\n",
    "    \n",
    "data[2234]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['this',\n",
       " 'is',\n",
       " 'reinforced',\n",
       " 'by',\n",
       " 'the',\n",
       " 'later',\n",
       " 'hospitalisation',\n",
       " 'and',\n",
       " 'feeding',\n",
       " 'problem']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('DataSet/Pickle/Books dataset.pickle', 'rb') as f:\n",
    "    data_books = pickle.load(f)\n",
    "    \n",
    "data_books[2234]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['she',\n",
       " 'is',\n",
       " 'in',\n",
       " 'an',\n",
       " 'acting',\n",
       " 'troupe',\n",
       " 'to',\n",
       " 'get',\n",
       " 'away',\n",
       " 'from',\n",
       " 'her',\n",
       " 'problem',\n",
       " 'she',\n",
       " 'witness',\n",
       " 'love',\n",
       " 'fear']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('DataSet/Pickle/Moviereviews.pickle', 'rb') as f:\n",
    "    data_movie = pickle.load(f)\n",
    "    \n",
    "data_movie[2234]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hmm',\n",
       " 'i',\n",
       " 'like',\n",
       " 'this',\n",
       " 'book',\n",
       " 'lol',\n",
       " 'newas',\n",
       " 'gota',\n",
       " 'go',\n",
       " 'watch',\n",
       " 'the',\n",
       " 'match',\n",
       " 'sarah',\n",
       " 'xxx',\n",
       " 'i',\n",
       " 'can',\n",
       " 't',\n",
       " 'wait',\n",
       " 'for',\n",
       " 'thing',\n",
       " 'to',\n",
       " 'get',\n",
       " 'moving',\n",
       " 'i',\n",
       " 'want',\n",
       " 'out',\n",
       " 'of',\n",
       " 'here']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('DataSet/Pickle/Blog Authorship Data.pickle', 'rb') as f:\n",
    "    data_blog = pickle.load(f)\n",
    "    \n",
    "data_blog[2234]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i', 'would', 'have', 'been', 'out', 'of', 'my', 'mind']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('DataSet/Pickle/Twitter New.pickle', 'rb') as f:\n",
    "    data_twitter = pickle.load(f)\n",
    "    \n",
    "data_twitter[2234]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_data = data\n",
    "random.seed(87)\n",
    "random.shuffle(tokenized_data)\n",
    "\n",
    "train_size = int(len(tokenized_data) * 0.8)\n",
    "train_data = tokenized_data[0:train_size]\n",
    "test_data = tokenized_data[train_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_data1 = data_books\n",
    "random.seed(87)\n",
    "random.shuffle(tokenized_data1)\n",
    "\n",
    "train_size_books = int(len(tokenized_data1) * 0.8)\n",
    "train_data_books = tokenized_data1[0:train_size_books]\n",
    "test_data_books = tokenized_data1[train_size_books:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_data2 = data_movie\n",
    "random.seed(87)\n",
    "random.shuffle(tokenized_data2)\n",
    "\n",
    "train_size_movie = int(len(tokenized_data2) * 0.8)\n",
    "train_data_movie = tokenized_data2[0:train_size_movie]\n",
    "test_data_movie = tokenized_data2[train_size_movie:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_data3 = data_blog\n",
    "random.seed(87)\n",
    "random.shuffle(tokenized_data3)\n",
    "\n",
    "train_size_blog = int(len(tokenized_data3) * 0.8)\n",
    "train_data_blog = tokenized_data3[0:train_size_blog]\n",
    "test_data_blog = tokenized_data3[train_size_blog:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_data4 = data_twitter\n",
    "random.seed(87)\n",
    "random.shuffle(tokenized_data4)\n",
    "\n",
    "train_size_twitter = int(len(tokenized_data4) * 0.8)\n",
    "train_data_twitter = tokenized_data4[0:train_size_twitter]\n",
    "test_data_twitter = tokenized_data4[train_size_twitter:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_words(tokenized_sentences):\n",
    "    \"\"\"\n",
    "    Count the number of word appearence in the tokenized sentences\n",
    "\n",
    "    Args:\n",
    "        tokenized_sentences: List of lists of strings\n",
    "\n",
    "    Returns:\n",
    "        dict that maps word (str) to the frequency (int)\n",
    "    \"\"\"\n",
    "\n",
    "    word_counts = {}\n",
    "    # Looping through each sentence\n",
    "    for sentence in tokenized_sentences:\n",
    "\n",
    "        for token in sentence:\n",
    "\n",
    "            if token not in word_counts.keys():\n",
    "                word_counts[token] = 1\n",
    "            else:\n",
    "                word_counts[token] += 1\n",
    "\n",
    "    return word_counts\n",
    "\n",
    "\n",
    "def get_words_with_nplus_frequency(tokenized_sentences, count_threshold):\n",
    "    \"\"\"\n",
    "    Find the words that appear count_threshold times or more\n",
    "\n",
    "    Args:\n",
    "        tokenized_sentences: List of lists of sentences\n",
    "        count_threshold: minimum number of occurrences for a word to be in the closed vocabulary.\n",
    "\n",
    "    Returns:\n",
    "        List of words that appear count_threshold times or more\n",
    "    \"\"\"\n",
    "    closed_vocab = []\n",
    "\n",
    "    # Using the function that you defined earlier to count the words\n",
    "    word_counts = count_words(tokenized_sentences)\n",
    "    # for each word and its count\n",
    "    for word, cnt in word_counts.items():\n",
    "\n",
    "        if cnt >= count_threshold:\n",
    "            closed_vocab.append(word)\n",
    "\n",
    "    return closed_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_oov_words_by_unk(tokenized_sentences, vocabulary, unknown_token=\"<unk>\"):\n",
    "    \"\"\"\n",
    "    Replace words not in the given vocabulary with '<unk>' token.\n",
    "\n",
    "    Args:\n",
    "        tokenized_sentences: List of lists of strings\n",
    "        vocabulary: List of strings that we will use\n",
    "        unknown_token: A string representing unknown (out-of-vocabulary) words\n",
    "\n",
    "    Returns:\n",
    "        List of lists of strings, with words not in the vocabulary replaced\n",
    "    \"\"\"\n",
    "    vocabulary = set(vocabulary)\n",
    "    replaced_tokenized_sentences = []\n",
    "    for sentence in tokenized_sentences:\n",
    "\n",
    "        # Initialize the list that will contain a single sentence with \"unknown_token\" replacements\n",
    "        replaced_sentence = []\n",
    "        for token in sentence:\n",
    "            if token in vocabulary:\n",
    "                # If so, append the word to the replaced_sentence\n",
    "                replaced_sentence.append(token)\n",
    "            else:\n",
    "                # otherwise, append the unknown token instead\n",
    "                replaced_sentence.append(unknown_token)\n",
    "        replaced_tokenized_sentences.append(replaced_sentence)\n",
    "\n",
    "    return replaced_tokenized_sentences\n",
    "\n",
    "\n",
    "def preprocess_data(train_data, test_data, count_threshold):\n",
    "    \"\"\"\n",
    "    Preprocess data, i.e.,\n",
    "        - Find tokens that appear at least N times in the training data.\n",
    "        - Replace tokens that appear less than N times by \"<unk>\" both for training and test data.\n",
    "    Args:\n",
    "        train_data, test_data: List of lists of strings.\n",
    "        count_threshold: Words whose count is less than this are\n",
    "                      treated as unknown.\n",
    "\n",
    "    Returns:\n",
    "        Tuple of\n",
    "        - training data with low frequent words replaced by \"<unk>\"\n",
    "        - test data with low frequent words replaced by \"<unk>\"\n",
    "        - vocabulary of words that appear n times or more in the training data\n",
    "    \"\"\"\n",
    "    vocabulary = get_words_with_nplus_frequency(train_data, count_threshold)\n",
    "    # For the train data, replace less common words with \"<unk>\"\n",
    "    train_data_replaced = replace_oov_words_by_unk(train_data, vocabulary)\n",
    "    # For the test data, replace less common words with \"<unk>\"\n",
    "    test_data_replaced = replace_oov_words_by_unk(test_data, vocabulary)\n",
    "\n",
    "    return train_data_replaced, test_data_replaced, vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# news data\n",
    "minimum_freq = 3\n",
    "train_data_processed, test_data_processed, vocabulary = preprocess_data(train_data, test_data, minimum_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "minimum_freq = 3\n",
    "train_data_processed_books, test_data_processed_books, vocabulary_books = preprocess_data(train_data_books, test_data_books,\n",
    "                                                                                          minimum_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "minimum_freq = 3\n",
    "train_data_processed_movie, test_data_processed_movie, vocabulary_movie = preprocess_data(train_data_movie, test_data_movie,\n",
    "                                                                                          minimum_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "minimum_freq = 3\n",
    "train_data_processed_blog, test_data_processed_blog, vocabulary_blog = preprocess_data(train_data_blog, test_data_blog,\n",
    "                                                                                          minimum_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "minimum_freq = 3\n",
    "train_data_processed_twitter, test_data_processed_twitter, vocabulary_twitter = preprocess_data(train_data_twitter, \n",
    "                                                                                                test_data_twitter, \n",
    "                                                                                                minimum_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Develop n-gram based language models\n",
    "def count_n_grams(data, n, start_token='<s>', end_token='<e>'):\n",
    "    \"\"\"\n",
    "    Count all n-grams in the data\n",
    "\n",
    "    Args:\n",
    "        data: List of lists of words\n",
    "        n: number of words in a sequence\n",
    "\n",
    "    Returns:\n",
    "        A dictionary that maps a tuple of n-words to its frequency\n",
    "    \"\"\"\n",
    "\n",
    "    # Initialize dictionary of n-grams and their counts\n",
    "    n_grams = {}\n",
    "\n",
    "    for sentence in data:\n",
    "\n",
    "        # prepend start token n times, and  append <e> one time\n",
    "        sentence = [start_token] * n + sentence + [end_token]\n",
    "        sentence = tuple(sentence)\n",
    "\n",
    "        m = len(sentence) if n == 1 else len(sentence) - 1\n",
    "        for i in range(m):\n",
    "\n",
    "            # Get the n-gram from i to i+n\n",
    "            n_gram = sentence[i:i + n]\n",
    "\n",
    "            # check if the n-gram is in the dictionary\n",
    "            if n_gram in n_grams.keys():\n",
    "\n",
    "                # Increment the count for this n-gram\n",
    "                n_grams[n_gram] += 1\n",
    "            else:\n",
    "                n_grams[n_gram] = 1\n",
    "\n",
    "    return n_grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def estimate_probability(word, previous_n_gram,\n",
    "                         n_gram_counts, n_plus1_gram_counts, vocabulary_size, k=1.0):\n",
    "    \"\"\"\n",
    "    Estimate the probabilities of a next word using the n-gram counts with k-smoothing\n",
    "\n",
    "    Args:\n",
    "        word: next word\n",
    "        previous_n_gram: A sequence of words of length n\n",
    "        n_gram_counts: Dictionary of counts of (n+1)-grams\n",
    "        n_plus1_gram_counts: Dictionary of counts of (n+1)-grams\n",
    "        vocabulary_size: number of words in the vocabulary\n",
    "        k: positive constant, smoothing parameter\n",
    "\n",
    "    Returns:\n",
    "        A probability\n",
    "    \"\"\"\n",
    "    # See notes for the formula implemented here.\n",
    "    temp = previous_n_gram\n",
    "    previous_n_gram = tuple(previous_n_gram)\n",
    "\n",
    "    # Set the denominator\n",
    "    # If the previous n-gram exists in the dictionary of n-gram counts,\n",
    "    # Get its count.  Otherwise set the count to zero\n",
    "    if previous_n_gram in n_gram_counts:\n",
    "        previous_n_gram_count = n_gram_counts[previous_n_gram]\n",
    "    else: # stupid backoff is done here.\n",
    "        previous_n_gram_count = 0 \n",
    "    # Calculate the denominator using the count of the previous n gram - applying k-smoothing\n",
    "    denominator = previous_n_gram_count + k * vocabulary_size\n",
    "\n",
    "    # Define n plus 1 gram as the previous n-gram plus the current word as a tuple\n",
    "    n_plus1_gram = previous_n_gram + (word,)\n",
    "    temp2 = list(n_plus1_gram)\n",
    "\n",
    "    # Set the count to the count in the dictionary,\n",
    "    # otherwise 0 if not in the dictionary\n",
    "    if n_plus1_gram in n_plus1_gram_counts:\n",
    "        n_plus1_gram_count = n_plus1_gram_counts[n_plus1_gram] \n",
    "    else:\n",
    "        n_plus1_gram_count = 0\n",
    "    # Define the numerator use the count of the n-gram plus current word - apply smoothing\n",
    "    numerator = n_plus1_gram_count + k\n",
    "    probability = numerator / denominator\n",
    "\n",
    "    return probability\n",
    "\n",
    "\n",
    "# Estimate probabilities for all words\n",
    "def estimate_probabilities(previous_n_gram, n_gram_counts, n_plus1_gram_counts, vocabulary, k=1.0):\n",
    "    \"\"\"\n",
    "    Estimate the probabilities of next words using the n-gram counts with k-smoothing\n",
    "\n",
    "    Args:\n",
    "        previous_n_gram: A sequence of words of length n\n",
    "        n_gram_counts: Dictionary of counts of (n+1)-grams\n",
    "        n_plus1_gram_counts: Dictionary of counts of (n+1)-grams\n",
    "        vocabulary: List of words\n",
    "        k: positive constant, smoothing parameter\n",
    "\n",
    "    Returns:\n",
    "        A dictionary mapping from next words to the probability.\n",
    "    \"\"\"\n",
    "\n",
    "    # convert list to tuple to use it as a dictionary key\n",
    "    previous_n_gram = tuple(previous_n_gram)\n",
    "\n",
    "    # add <e> <unk> to the vocabulary\n",
    "    # <s> is not needed since it should not appear as the next word\n",
    "    vocabulary = vocabulary + [\"<e>\", \"<unk>\"]\n",
    "    vocabulary_size = len(vocabulary)\n",
    "\n",
    "    probabilities = {}\n",
    "    for word in vocabulary:\n",
    "        probability = estimate_probability(word, previous_n_gram,\n",
    "                                           n_gram_counts, n_plus1_gram_counts,\n",
    "                                           vocabulary_size, k=k)\n",
    "        probabilities[word] = probability\n",
    "\n",
    "    return probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perplexity - It is used as a metric to see how accurate the model is.\n",
    "def calculate_perplexity(sentence, n_gram_counts, n_plus1_gram_counts, vocabulary_size, k=1.0):\n",
    "    \"\"\"\n",
    "    Calculate perplexity for a list of sentences\n",
    "\n",
    "    Args:\n",
    "        sentence: List of strings\n",
    "        n_gram_counts: Dictionary of counts of (n+1)-grams\n",
    "        n_plus1_gram_counts: Dictionary of counts of (n+1)-grams\n",
    "        vocabulary_size: number of unique words in the vocabulary\n",
    "        k: Positive smoothing constant\n",
    "\n",
    "    Returns:\n",
    "        Perplexity score\n",
    "    \"\"\"\n",
    "    # length of previous words\n",
    "    n = len(list(n_gram_counts.keys())[0])\n",
    "\n",
    "    # prepend <s> and append <e>\n",
    "    sentence = [\"<s>\"] * n + sentence + [\"<e>\"]\n",
    "    sentence = tuple(sentence)\n",
    "    N = len(sentence)\n",
    "\n",
    "    product_pi = 1.0\n",
    "\n",
    "    for t in range(n, N):\n",
    "        n_gram = sentence[t - n:t]\n",
    "        # get the word at position t\n",
    "        word = sentence[t]\n",
    "\n",
    "        # Estimate the probability of the word\n",
    "        probability = estimate_probability(word, n_gram, n_gram_counts, n_plus1_gram_counts, len(unique_words), k=1)\n",
    "\n",
    "        # This 'product_pi' is a cumulative product\n",
    "        # of the (1/P) factors that are calculated in the loop\n",
    "        product_pi *= 1 / probability\n",
    "\n",
    "    # Take the Nth root of the product\n",
    "    perplexity = product_pi ** (1 / float(N))\n",
    "\n",
    "    return perplexity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build an auto-complete system\n",
    "def suggest_a_word(previous_tokens, n_gram_counts, n_plus1_gram_counts, vocabulary, start_with, k=1.0):\n",
    "    \"\"\"\n",
    "    Get suggestion for the next word\n",
    "\n",
    "    Args:\n",
    "        previous_tokens: The sentence you input where each token is a word. Must have length > n\n",
    "        n_gram_counts: Dictionary of counts of (n)-grams\n",
    "        n_plus1_gram_counts: Dictionary of counts of (n+1)-grams\n",
    "        vocabulary: List of words\n",
    "        k: positive constant, smoothing parameter\n",
    "        start_with: If not None, specifies the first few letters of the next word\n",
    "\n",
    "    Returns:\n",
    "        A tuple of\n",
    "          - string of the most likely next word\n",
    "          - corresponding probability\n",
    "    \"\"\"\n",
    "    n = len(list(n_gram_counts.keys())[0])\n",
    "\n",
    "    # From the words that the user already typed get the most recent 'n' words as the previous n-gram\n",
    "    previous_n_gram = previous_tokens[-n:]\n",
    "\n",
    "    # Estimate the probabilities that each word in the vocabulary is the next word\n",
    "    probabilities = estimate_probabilities(previous_n_gram,\n",
    "                                           n_gram_counts, n_plus1_gram_counts,\n",
    "                                           vocabulary, k=k)\n",
    "\n",
    "    suggestion = None\n",
    "    max_prob = 0\n",
    "    for word, prob in probabilities.items():\n",
    "\n",
    "        # If the optional start_with string is set\n",
    "        if start_with != None:\n",
    "            if not word.startswith(start_with):\n",
    "                # If so, don't consider this word (move onto the next word)\n",
    "                continue\n",
    "\n",
    "        if prob > max_prob:\n",
    "            # If so, save this word as the best suggestion (so far)\n",
    "            suggestion = word\n",
    "\n",
    "            max_prob = prob\n",
    "\n",
    "    return suggestion, max_prob\n",
    "\n",
    "\n",
    "# Get multiple suggetions\n",
    "def get_suggestions(previous_tokens, n_gram_counts_list, vocabulary, start_with, k=1.0):\n",
    "    model_counts = len(n_gram_counts_list)\n",
    "    suggestions = []\n",
    "    for i in range(model_counts - 1):\n",
    "        n_gram_counts = n_gram_counts_list[i]\n",
    "        n_plus1_gram_counts = n_gram_counts_list[i + 1]\n",
    "\n",
    "        suggestion = suggest_a_word(previous_tokens, n_gram_counts,\n",
    "                                    n_plus1_gram_counts, vocabulary,\n",
    "                                    k=k, start_with=start_with)\n",
    "        suggestions.append(suggestion)\n",
    "    return suggestions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = train_data_processed #+ test_data_processed\n",
    "unique_words = vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences_books = train_data_processed_books\n",
    "unique_words_books = vocabulary_books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences_movie = train_data_processed_movie\n",
    "unique_words_movie = vocabulary_movie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences_blog = train_data_processed_blog\n",
    "unique_words_blog = vocabulary_blog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences_twitter = train_data_processed_twitter\n",
    "unique_words_twitter = vocabulary_twitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73946"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(unique_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# news dataset\n",
    "unigram_counts = count_n_grams(sentences, 1)\n",
    "bigram_counts = count_n_grams(sentences, 2)\n",
    "trigram_counts = count_n_grams(sentences, 3)\n",
    "quadgram_counts = count_n_grams(sentences, 4)\n",
    "# qintgram_counts = count_n_grams(sentences, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "unigram_counts_books = count_n_grams(sentences_books, 1)\n",
    "bigram_counts_books = count_n_grams(sentences_books, 2)\n",
    "trigram_counts_books = count_n_grams(sentences_books, 3)\n",
    "quadgram_counts_books = count_n_grams(sentences_books, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "unigram_counts_movie = count_n_grams(sentences_movie, 1)\n",
    "bigram_counts_movie = count_n_grams(sentences_movie, 2)\n",
    "trigram_counts_movie = count_n_grams(sentences_movie, 3)\n",
    "quadgram_counts_movie = count_n_grams(sentences_movie, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "unigram_counts_blog = count_n_grams(sentences_blog, 1)\n",
    "bigram_counts_blog = count_n_grams(sentences_blog, 2)\n",
    "trigram_counts_blog = count_n_grams(sentences_blog, 3)\n",
    "quadgram_counts_blog = count_n_grams(sentences_blog, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "unigram_counts_twitter = count_n_grams(sentences_twitter, 1)\n",
    "bigram_counts_twitter = count_n_grams(sentences_twitter, 2)\n",
    "trigram_counts_twitter = count_n_grams(sentences_twitter, 3)\n",
    "quadgram_counts_twitter = count_n_grams(sentences_twitter, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3000\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "print(sys.getrecursionlimit())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('not', 0.0011250827266710787), ('metallic', 4.0566854175681525e-05)]\n"
     ]
    }
   ],
   "source": [
    "n_gram_counts_list = [bigram_counts, trigram_counts, quadgram_counts]\n",
    "previous_tokens = ['try', 'to', 'pick', 'floor', 'and', 'wall', 'color', 'that', 'are'] # warm\n",
    "start_withs = None\n",
    "tmp_suggest4 = get_suggestions(previous_tokens, n_gram_counts_list, unique_words, start_withs, k=1.0)\n",
    "\n",
    "print(tmp_suggest4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15799908\n"
     ]
    }
   ],
   "source": [
    "f = open(\"DataSet/News Mixed.txt\", \"r\", encoding=\"UTF-8\")\n",
    "f = f.read()\n",
    "words = f.split()\n",
    "print(len(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1504727a2b914ca2a923d38b27bf586b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "44\n"
     ]
    }
   ],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts, bigram_counts, trigram_counts, quadgram_counts]\n",
    "start_withs = None\n",
    "count = 0\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed[i][0:len(test_data_processed[i])-1]\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words, start_withs, k=1.0)\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed[i][len(test_data_processed[i])-1] == j[0]:\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4, 3, 2, 5, 1]\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "l = [1,2,3,4,5]\n",
    "random.shuffle(l)\n",
    "print(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8838f408d456478b8fb818b9d2f08dab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "13\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_books, etc]\n",
    "# testing MovieReviews - test_data_processed_movie\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_books, bigram_counts_books, trigram_counts_books, quadgram_counts_books]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_movie) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_movie[i][0:len(test_data_processed_movie[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_books, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_movie[i][len(test_data_processed_movie[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac0119d6f1dd451385de0ae9b066c97c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "6\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_books, etc]\n",
    "# testing MovieReviews - test_data_processed_twitter\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_books, bigram_counts_books, trigram_counts_books, quadgram_counts_books]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_twitter) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_twitter[i][0:len(test_data_processed_twitter[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_books, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_twitter[i][len(test_data_processed_twitter[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6708588d00f5430aa9cb3b55d67e22cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "12\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_books, etc]\n",
    "# testing MovieReviews - test_data_processed\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_books, bigram_counts_books, trigram_counts_books, quadgram_counts_books]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed[i][0:len(test_data_processed[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_books, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed[i][len(test_data_processed[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b1074ee7d5d4b8eb7513a8c7fb1195c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "13\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_books, etc]\n",
    "# testing MovieReviews - test_data_processed_blog\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_books, bigram_counts_books, trigram_counts_books, quadgram_counts_books]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_blog) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_blog[i][0:len(test_data_processed_blog[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_books, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_blog[i][len(test_data_processed_blog[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf2f0c71f7784ee684ef04830568bf59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_movie, etc]\n",
    "# testing MovieReviews - test_data_processed_blog\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_movie, bigram_counts_movie, trigram_counts_movie, quadgram_counts_movie]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_books) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_books[i][0:len(test_data_processed_books[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_movie, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_books[i][len(test_data_processed_books[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0ece0ff72eb47fca7f9674895f7d79b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "21\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_movie, etc]\n",
    "# testing MovieReviews - test_data_processed_twitter\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_movie, bigram_counts_movie, trigram_counts_movie, quadgram_counts_movie]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_twitter) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_twitter[i][0:len(test_data_processed_twitter[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_movie, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_twitter[i][len(test_data_processed_twitter[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d01b0eb6b8434fdda254242026953498",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "5\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_movie, etc]\n",
    "# testing MovieReviews - test_data_processed\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_movie, bigram_counts_movie, trigram_counts_movie, quadgram_counts_movie]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed) #\n",
    "\n",
    "for i in tqdm(range(1100, 1200)):\n",
    "    testing = test_data_processed[i][0:len(test_data_processed[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_movie, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed[i][len(test_data_processed[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c3fef6e86cb41df94b775e720bcebff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "6\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_movie, etc]\n",
    "# testing MovieReviews - test_data_processed_blog\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_movie, bigram_counts_movie, trigram_counts_movie, quadgram_counts_movie]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_blog) #\n",
    "\n",
    "for i in tqdm(range(100, 200)):\n",
    "    testing = test_data_processed_blog[i][0:len(test_data_processed_blog[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_movie, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_blog[i][len(test_data_processed_blog[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d2e3776ed944aca93bc94803d67868b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_twitter, etc]\n",
    "# testing MovieReviews - test_data_processed_books\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_twitter, bigram_counts_twitter, trigram_counts_twitter, quadgram_counts_twitter]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_books) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_books[i][0:len(test_data_processed_books[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_twitter, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_books[i][len(test_data_processed_books[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a3cb7aebc604089b8a0b3a319247dff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "10\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_twitter, etc]\n",
    "# testing MovieReviews - test_data_processed_movie\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_twitter, bigram_counts_twitter, trigram_counts_twitter, quadgram_counts_twitter]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_movie) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_movie[i][0:len(test_data_processed_movie[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_twitter, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_movie[i][len(test_data_processed_movie[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5f12c1d33394b469065e004f45a6eb1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "6\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_twitter, etc]\n",
    "# testing MovieReviews - test_data_processed\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_twitter, bigram_counts_twitter, trigram_counts_twitter, quadgram_counts_twitter]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed[i][0:len(test_data_processed[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_twitter, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed[i][len(test_data_processed[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39bdcb125c034f4299a018ce6844d92b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "19\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_twitter, etc]\n",
    "# testing MovieReviews - test_data_processed_blog\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_twitter, bigram_counts_twitter, trigram_counts_twitter, quadgram_counts_twitter]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_blog) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_blog[i][0:len(test_data_processed_blog[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_twitter, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_blog[i][len(test_data_processed_blog[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c751810b7ff48a2b257fd99c90a0269",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "17\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts, etc]\n",
    "# testing MovieReviews - test_data_processed_books\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts, bigram_counts, trigram_counts, quadgram_counts]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_books) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_books[i][0:len(test_data_processed_books[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_books[i][len(test_data_processed_books[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "575ad712bd0d4b37963a3be7cec155b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "14\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts, etc]\n",
    "# testing MovieReviews - test_data_processed_movie\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts, bigram_counts, trigram_counts, quadgram_counts]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_movie) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_movie[i][0:len(test_data_processed_movie[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_movie[i][len(test_data_processed_movie[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "127af91acaa842f9adccbd37cdee1559",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "18\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts, etc]\n",
    "# testing MovieReviews - test_data_processed_movie\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts, bigram_counts, trigram_counts, quadgram_counts]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_twitter) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_twitter[i][0:len(test_data_processed_twitter[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_twitter[i][len(test_data_processed_twitter[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44991811f4cd49b3a54e04a8d5a3b88c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "6\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts, etc]\n",
    "# testing MovieReviews - test_data_processed_movie\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts, bigram_counts, trigram_counts, quadgram_counts]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_blog) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_blog[i][0:len(test_data_processed_blog[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_blog[i][len(test_data_processed_blog[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a83a4e8a2ed42a2841876f8bdd8d6b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "17\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_blog, etc]\n",
    "# testing MovieReviews - test_data_processed_books\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_blog, bigram_counts_blog, trigram_counts_blog, quadgram_counts_blog]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_books) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_books[i][0:len(test_data_processed_books[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_blog, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_books[i][len(test_data_processed_books[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f8b2c49a2bf84dc29f0ce19fe356962d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "12\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_blog, etc]\n",
    "# testing MovieReviews - test_data_processed_movie\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_blog, bigram_counts_blog, trigram_counts_blog, quadgram_counts_blog]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_movie) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_movie[i][0:len(test_data_processed_movie[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_blog, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_movie[i][len(test_data_processed_movie[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "edc00b928f094f77818a9c2ff7c837fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "12\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_blog, etc]\n",
    "# testing MovieReviews - test_data_processed_twitter\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_blog, bigram_counts_blog, trigram_counts_blog, quadgram_counts_blog]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed_twitter) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_twitter[i][0:len(test_data_processed_twitter[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_blog, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_twitter[i][len(test_data_processed_twitter[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ec8c8877f904e84a8f00bfaa71f38ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "7\n"
     ]
    }
   ],
   "source": [
    "# training Books dataset - [unigram_counts_blog, etc]\n",
    "# testing MovieReviews - test_data_processed\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "n_gram_counts_list = [unigram_counts_blog, bigram_counts_blog, trigram_counts_blog, quadgram_counts_blog]\n",
    "start_withs = None\n",
    "count = 0\n",
    "random.shuffle(test_data_processed) #\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed[i][0:len(test_data_processed[i])-1] #\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_blog, start_withs, k=1.0) #\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed[i][len(test_data_processed[i])-1] == j[0]: #\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8a16477f6f54b139594947198d33d2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "29\n"
     ]
    }
   ],
   "source": [
    "n_gram_counts_list = [unigram_counts_books, bigram_counts_books, trigram_counts_books, quadgram_counts_books]\n",
    "start_withs = None\n",
    "count = 0\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_books[i][0:len(test_data_processed_books[i])-1]\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_books, start_withs, k=1.0)\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_books[i][len(test_data_processed_books[i])-1] == j[0]:\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74b9b73da028483b8b56d318eac2a344",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "38\n"
     ]
    }
   ],
   "source": [
    "n_gram_counts_list = [unigram_counts_movie, bigram_counts_movie, trigram_counts_movie, quadgram_counts_movie]\n",
    "start_withs = None\n",
    "count = 0\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_movie[i][0:len(test_data_processed_movie[i])-1]\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_movie, start_withs, k=1.0)\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_movie[i][len(test_data_processed_movie[i])-1] == j[0]:\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cecb1450bfe34a8fa42fe13bb6df3b6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "28\n"
     ]
    }
   ],
   "source": [
    "n_gram_counts_list = [unigram_counts_twitter, bigram_counts_twitter, trigram_counts_twitter, quadgram_counts_twitter]\n",
    "start_withs = None\n",
    "count = 0\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_twitter[i][0:len(test_data_processed_twitter[i])-1]\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_twitter, start_withs, k=1.0)\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_twitter[i][len(test_data_processed_twitter[i])-1] == j[0]:\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78ab318f436649819729e73bd62164e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "18\n"
     ]
    }
   ],
   "source": [
    "n_gram_counts_list = [unigram_counts_blog, bigram_counts_blog, trigram_counts_blog, quadgram_counts_blog]\n",
    "start_withs = None\n",
    "count = 0\n",
    "\n",
    "for i in tqdm(range(0, 100)):\n",
    "    testing = test_data_processed_blog[i][0:len(test_data_processed_blog[i])-1]\n",
    "    # print(testing)\n",
    "    tmp_suggest = get_suggestions(testing, n_gram_counts_list, unique_words_blog, start_withs, k=1.0)\n",
    "    for j in tmp_suggest:\n",
    "        if test_data_processed_blog[i][len(test_data_processed_blog[i])-1] == j[0]:\n",
    "            count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}