notebooks/STTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gsdmm import MovieGroupProcess\n",
    "from os import path\n",
    "import gensim\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In stage 0: transferred 10402 clusters with 20 clusters populated\n"
     ]
    }
   ],
   "source": [
    "mgp = MovieGroupProcess(K=20, alpha=0.1, beta=0.1, n_iters=1)\n",
    "\n",
    "text = []\n",
    "with open(path.expanduser('~/work/data/20NG.txt'),\"r+\") as datafile:\n",
    "    while True:\n",
    "        vec = datafile.readline()\n",
    "\n",
    "        if not vec:\n",
    "            break\n",
    "\n",
    "        text.append(vec)\n",
    "\n",
    "# Transform documents\n",
    "tokens = [doc.split() for doc in text]\n",
    "\n",
    "id2word = gensim.corpora.Dictionary(tokens)\n",
    "\n",
    "corpus = [id2word.doc2bow(doc) for doc in tokens]\n",
    "\n",
    "y = mgp.fit(tokens, len(id2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents per topic : [ 237  580  408  312  202  223 1156  361  593  216  287  324  283  184\n",
      "  261  147  466  340  278 4456]\n",
      "********************\n",
      "Most important clusters (by number of docs inside): [19  6  8  1 16  2  7 17 11  3 10 12 18 14  0  5  9  4 13 15]\n",
      "********************\n",
      "Cluster 0 : [('write', 208), ('article', 165), ('one', 99), ('say', 96), ('get', 93), ('know', 87), ('think', 80), ('like', 69), ('make', 63), ('people', 57)]\n",
      "Cluster 1 : [('write', 392), ('car', 357), ('article', 299), ('get', 270), ('bike', 254), ('one', 243), ('like', 209), ('look', 178), ('know', 176), ('good', 167)]\n",
      "Cluster 2 : [('write', 377), ('article', 293), ('one', 156), ('get', 138), ('know', 130), ('think', 125), ('like', 110), ('say', 109), ('time', 100), ('make', 99)]\n",
      "Cluster 3 : [('write', 209), ('article', 150), ('one', 110), ('know', 104), ('get', 100), ('like', 85), ('think', 75), ('game', 71), ('say', 69), ('time', 66)]\n",
      "Cluster 4 : [('write', 139), ('article', 99), ('get', 86), ('know', 81), ('one', 77), ('like', 75), ('make', 67), ('orbit', 62), ('use', 53), ('think', 50)]\n",
      "Cluster 5 : [('drive', 352), ('disk', 157), ('write', 150), ('get', 95), ('article', 89), ('floppy', 84), ('like', 78), ('one', 78), ('system', 75), ('know', 74)]\n",
      "Cluster 6 : [('card', 668), ('get', 579), ('windows', 497), ('write', 494), ('use', 464), ('thank', 437), ('problem', 430), ('one', 426), ('know', 424), ('work', 415)]\n",
      "Cluster 7 : [('write', 212), ('get', 144), ('article', 132), ('mail', 120), ('use', 117), ('know', 115), ('thank', 113), ('need', 111), ('windows', 104), ('one', 101)]\n",
      "Cluster 8 : [('game', 420), ('write', 417), ('team', 307), ('article', 305), ('get', 227), ('play', 207), ('one', 204), ('year', 196), ('think', 178), ('win', 164)]\n",
      "Cluster 9 : [('write', 150), ('article', 108), ('get', 83), ('know', 82), ('use', 71), ('one', 68), ('widget', 61), ('say', 60), ('like', 55), ('think', 51)]\n",
      "Cluster 10 : [('write', 191), ('article', 112), ('get', 100), ('know', 99), ('thank', 84), ('one', 82), ('look', 80), ('graphics', 79), ('like', 77), ('need', 74)]\n",
      "Cluster 11 : [('write', 219), ('article', 153), ('one', 129), ('get', 84), ('know', 83), ('good', 82), ('like', 76), ('use', 73), ('system', 71), ('encryption', 65)]\n",
      "Cluster 12 : [('window', 140), ('write', 133), ('get', 99), ('thank', 89), ('know', 85), ('article', 79), ('one', 79), ('like', 76), ('program', 73), ('windows', 72)]\n",
      "Cluster 13 : [('write', 131), ('one', 106), ('article', 93), ('get', 89), ('know', 64), ('like', 63), ('say', 53), ('think', 46), ('make', 44), ('see', 40)]\n",
      "Cluster 14 : [('key', 245), ('write', 166), ('get', 123), ('chip', 122), ('one', 116), ('know', 114), ('article', 110), ('window', 109), ('use', 104), ('like', 89)]\n",
      "Cluster 15 : [('write', 108), ('article', 73), ('get', 66), ('think', 53), ('one', 49), ('say', 49), ('make', 39), ('like', 37), ('know', 36), ('good', 31)]\n",
      "Cluster 16 : [('file', 396), ('write', 277), ('get', 219), ('windows', 206), ('know', 178), ('program', 178), ('article', 177), ('thank', 165), ('work', 141), ('one', 135)]\n",
      "Cluster 17 : [('write', 337), ('article', 210), ('say', 173), ('one', 152), ('god', 129), ('get', 115), ('know', 113), ('morality', 93), ('like', 87), ('people', 85)]\n",
      "Cluster 18 : [('write', 198), ('article', 165), ('one', 131), ('get', 118), ('know', 100), ('like', 86), ('think', 68), ('see', 65), ('make', 63), ('good', 59)]\n",
      "Cluster 19 : [('one', 6924), ('write', 5732), ('say', 5635), ('get', 5298), ('people', 5081), ('max', 4581), ('make', 4521), ('think', 4483), ('know', 4348), ('like', 4008)]\n"
     ]
    }
   ],
   "source": [
    "doc_count = np.array(mgp.cluster_doc_count)\n",
    "print('Number of documents per topic :', doc_count)\n",
    "print('*'*20)\n",
    "# Topics sorted by the number of document they are allocated to\n",
    "top_index = doc_count.argsort()[::-1]\n",
    "print('Most important clusters (by number of docs inside):', top_index)\n",
    "print('*'*20)\n",
    "# Show the top 5 words in term frequency for each cluster \n",
    "for i in range(20):\n",
    "    print(\"Cluster\",i,':',sorted(mgp.cluster_word_distribution[i].items(), key=lambda item: item[1], reverse = True)[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>wikipedia-23885690</td>\n",
       "      <td>Research Design and Standards Organization  T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>wikipedia-23885928</td>\n",
       "      <td>The Death of Bunny Munro  The Death of Bunny ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>wikipedia-23886057</td>\n",
       "      <td>Management of prostate cancer  Treatment for ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>wikipedia-23886425</td>\n",
       "      <td>Cheetah reintroduction in India  Reintroducti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>wikipedia-23886491</td>\n",
       "      <td>Langtang National Park  The Langtang National...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0                                                  1\n",
       "0  wikipedia-23885690   Research Design and Standards Organization  T...\n",
       "1  wikipedia-23885928   The Death of Bunny Munro  The Death of Bunny ...\n",
       "2  wikipedia-23886057   Management of prostate cancer  Treatment for ...\n",
       "3  wikipedia-23886425   Cheetah reintroduction in India  Reintroducti...\n",
       "4  wikipedia-23886491   Langtang National Park  The Langtang National..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ref.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Topic\n",
      "\n",
      "write 0.017900172117039585\n",
      "article 0.014199655765920827\n",
      "one 0.008519793459552496\n",
      "say 0.008261617900172116\n",
      "get 0.008003442340791738\n",
      "know 0.007487091222030981\n",
      "think 0.0068846815834767644\n",
      "like 0.005938037865748709\n",
      "make 0.005421686746987952\n",
      "people 0.0049053356282271945\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.013770330558190186\n",
      "car 0.012540836758351775\n",
      "article 0.010503389890048126\n",
      "get 0.0094846664558963\n",
      "bike 0.008922612147398742\n",
      "one 0.00853619981030667\n",
      "like 0.007341834404749359\n",
      "look 0.0062528541820353395\n",
      "know 0.0061825973934731444\n",
      "good 0.005866441844943267\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.019244512506380807\n",
      "article 0.014956610515569168\n",
      "one 0.007963246554364471\n",
      "get 0.007044410413476264\n",
      "know 0.006636038795303726\n",
      "think 0.006380806533945891\n",
      "like 0.005615109749872384\n",
      "say 0.0055640632976008166\n",
      "time 0.005104645227156712\n",
      "make 0.005053598774885145\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.01563903023046992\n",
      "article 0.011224184375935349\n",
      "one 0.00823106854235259\n",
      "know 0.007782101167315175\n",
      "get 0.0074827895839568994\n",
      "like 0.006360371146363364\n",
      "think 0.005612092187967674\n",
      "game 0.005312780604609399\n",
      "say 0.00516312481293026\n",
      "time 0.004938641125411553\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.013321832470768642\n",
      "article 0.00948821161587119\n",
      "get 0.008242284838029518\n",
      "know 0.007763082231167337\n",
      "one 0.007379720145677593\n",
      "like 0.00718803910293272\n",
      "make 0.00642131493195323\n",
      "orbit 0.005942112325091048\n",
      "use 0.005079547632739122\n",
      "think 0.0047920260686218135\n",
      "\n",
      "Topic\n",
      "\n",
      "drive 0.03201164059658058\n",
      "disk 0.014277919243361222\n",
      "write 0.01364132411786104\n",
      "get 0.008639505274645326\n",
      "article 0.008093852309930883\n",
      "floppy 0.007639141506002182\n",
      "like 0.007093488541287741\n",
      "one 0.007093488541287741\n",
      "system 0.00682066205893052\n",
      "know 0.00672971989814478\n",
      "\n",
      "Topic\n",
      "\n",
      "card 0.012347732860126805\n",
      "get 0.010702600787445239\n",
      "windows 0.009186861124974584\n",
      "write 0.009131407234884193\n",
      "use 0.008576868333980296\n",
      "thank 0.008077783323166787\n",
      "problem 0.007948390912955877\n",
      "one 0.007874452392835357\n",
      "know 0.007837483132775097\n",
      "work 0.007671121462503928\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.012741916095684577\n",
      "get 0.00865488640461594\n",
      "article 0.007933645870897944\n",
      "mail 0.00721240533717995\n",
      "use 0.007032095203750451\n",
      "know 0.006911888448130785\n",
      "thank 0.006791681692511119\n",
      "need 0.0066714749368914535\n",
      "windows 0.0062507512922226225\n",
      "one 0.006070441158793124\n",
      "\n",
      "Topic\n",
      "\n",
      "game 0.014111480697510332\n",
      "write 0.01401068440681383\n",
      "team 0.010314820414608742\n",
      "article 0.01024762288747774\n",
      "get 0.007626919329368679\n",
      "play 0.006954944058058663\n",
      "one 0.006854147767362161\n",
      "year 0.006585357658838155\n",
      "think 0.005980579914659141\n",
      "win 0.005510197224742129\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.014886859865025803\n",
      "article 0.01071853910281858\n",
      "get 0.008237395791980944\n",
      "know 0.00813815005954744\n",
      "use 0.00704644700277888\n",
      "one 0.006748709805478364\n",
      "widget 0.006053989678443827\n",
      "say 0.0059547439460103215\n",
      "like 0.0054585152838427945\n",
      "think 0.005061532354108773\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.014863813229571985\n",
      "article 0.008715953307392997\n",
      "get 0.007782101167315175\n",
      "know 0.0077042801556420236\n",
      "thank 0.006536964980544747\n",
      "one 0.006381322957198444\n",
      "look 0.0062256809338521405\n",
      "graphics 0.006147859922178988\n",
      "like 0.005992217898832685\n",
      "need 0.00575875486381323\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.014782315221059737\n",
      "article 0.010327370907863651\n",
      "one 0.00870739115761053\n",
      "get 0.0056699291258859265\n",
      "know 0.00560242996962538\n",
      "good 0.005534930813364833\n",
      "like 0.005129935875801552\n",
      "use 0.004927438407019912\n",
      "system 0.004792440094498819\n",
      "encryption 0.004387445156935539\n",
      "\n",
      "Topic\n",
      "\n",
      "window 0.011482939632545932\n",
      "write 0.010908792650918635\n",
      "get 0.00812007874015748\n",
      "thank 0.0072998687664042\n",
      "know 0.0069717847769028875\n",
      "article 0.006479658792650919\n",
      "one 0.006479658792650919\n",
      "like 0.006233595800524934\n",
      "program 0.0059875328083989505\n",
      "windows 0.005905511811023622\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.014966297269507597\n",
      "one 0.012110133668456528\n",
      "article 0.010624928595909973\n",
      "get 0.010167942419741803\n",
      "know 0.007311778818690735\n",
      "like 0.007197532274648692\n",
      "say 0.006055066834228264\n",
      "think 0.005255341025933965\n",
      "make 0.00502684793784988\n",
      "see 0.004569861761681709\n",
      "\n",
      "Topic\n",
      "\n",
      "key 0.018210197710718003\n",
      "write 0.01233833804073138\n",
      "get 0.009142262524156385\n",
      "chip 0.009067935186561618\n",
      "one 0.008621971160993013\n",
      "know 0.008473316485803479\n",
      "article 0.008176007135424409\n",
      "window 0.008101679797829641\n",
      "use 0.007730043109855805\n",
      "like 0.006615133045934295\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.01565217391304348\n",
      "article 0.010579710144927536\n",
      "get 0.009565217391304347\n",
      "think 0.007681159420289855\n",
      "one 0.007101449275362319\n",
      "say 0.007101449275362319\n",
      "make 0.005652173913043478\n",
      "like 0.0053623188405797105\n",
      "know 0.0052173913043478265\n",
      "good 0.004492753623188406\n",
      "\n",
      "Topic\n",
      "\n",
      "file 0.01893106415527297\n",
      "write 0.013242183765178316\n",
      "get 0.010469452146476719\n",
      "windows 0.009847977818147051\n",
      "know 0.008509417726360073\n",
      "program 0.008509417726360073\n",
      "article 0.008461612008796252\n",
      "thank 0.007887943398030405\n",
      "work 0.006740606176498709\n",
      "one 0.006453771871115785\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.020115800155196085\n",
      "article 0.012535068345967886\n",
      "say 0.01032650868501164\n",
      "one 0.00907300185041485\n",
      "god 0.00770011341252313\n",
      "get 0.006864442189458604\n",
      "know 0.006745060586163672\n",
      "morality 0.00555124455321435\n",
      "like 0.005193099743329553\n",
      "people 0.005073718140034621\n",
      "\n",
      "Topic\n",
      "\n",
      "write 0.013748090542980141\n",
      "article 0.011456742119150118\n",
      "one 0.009095958894597973\n",
      "get 0.008193306485210387\n",
      "know 0.006943480072212193\n",
      "like 0.005971392862102486\n",
      "think 0.004721566449104291\n",
      "see 0.004513262046937925\n",
      "make 0.004374392445493681\n",
      "good 0.004096653242605193\n",
      "\n",
      "Topic\n",
      "\n",
      "one 0.007287991444714838\n",
      "write 0.006033328561684785\n",
      "say 0.005931229317008682\n",
      "get 0.005576513384474178\n",
      "people 0.005348105795868875\n",
      "max 0.004821821029497209\n",
      "make 0.004758666857532608\n",
      "think 0.004718669215288362\n",
      "know 0.004576572328368012\n",
      "like 0.004218698687235279\n"
     ]
    }
   ],
   "source": [
    "for topic in mgp.cluster_word_distribution:\n",
    "    total = sum(topic.values())\n",
    "    print('\\nTopic\\n')\n",
    "    for word, freq in sorted(topic.items(), key=lambda item: item[1], reverse = True)[:10]:\n",
    "        print(word,freq/total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}