Bag_of_words.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example\n",
    "<strong>Review 1:</strong> This movie is very scary and long <br/>\n",
    "<strong>Review 2:</strong> This movie is not scary and is slow <br/>\n",
    "<strong>Review 3:</strong> This movie is spooky and good <br/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "allsentences = [\"This movie is very scary and long\", \"This movie is not scary and is slow\", \"This movie is spooky and good\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenize a sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_extraction(sentence):    \n",
    "    words = sentence.split()    \n",
    "    words = [w.lower() for w in words]    \n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['this', 'movie', 'is', 'very', 'scary', 'and', 'long']"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_extraction(allsentences[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(sentences):    \n",
    "    words = []    \n",
    "    for sentence in sentences:        \n",
    "        w = word_extraction(sentence)        \n",
    "        words.extend(w)            \n",
    "    words = sorted(list(set(words)))    \n",
    "    return words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate the vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_bow(allsentences):        \n",
    "    vocab = sorted(tokenize(allsentences))\n",
    "    return vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the vocabulary : \n",
      "['and', 'good', 'is', 'long', 'movie', 'not', 'scary', 'slow', 'spooky', 'this', 'very']\n"
     ]
    }
   ],
   "source": [
    "vocab = generate_bow(allsentences)\n",
    "print(\"the vocabulary : \\n{0}\".format(vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix_representation = np.zeros((len(allsentences),len(vocab)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text Representation : \n",
      "[[1. 0. 1. 1. 1. 0. 1. 0. 0. 1. 1.]\n",
      " [1. 0. 2. 0. 1. 1. 1. 1. 0. 1. 0.]\n",
      " [1. 1. 1. 0. 1. 0. 0. 0. 1. 1. 0.]]\n",
      "\n",
      "['This movie is very scary and long', 'This movie is not scary and is slow', 'This movie is spooky and good']\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for j,sentence in enumerate(allsentences):\n",
    "    words = word_extraction(sentence)\n",
    "    for word in words:\n",
    "        index = vocab.index(word)\n",
    "        matrix_representation[j,index] += 1 \n",
    "\n",
    "print(\"Text Representation : \")\n",
    "print(\"{0}\\n\".format(np.array(matrix_representation)))\n",
    "print(allsentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'and': 0,\n",
       " 'good': 1,\n",
       " 'is': 2,\n",
       " 'long': 3,\n",
       " 'movie': 4,\n",
       " 'not': 5,\n",
       " 'scary': 6,\n",
       " 'slow': 7,\n",
       " 'spooky': 8,\n",
       " 'this': 9,\n",
       " 'very': 10}"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(zip(vocab,range(len(vocab))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bag-of-words in sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1],\n",
       "       [1, 0, 2, 0, 1, 1, 1, 1, 0, 1, 0],\n",
       "       [1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0]], dtype=int64)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(allsentences).toarray()\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}