notebooks/training_corpus_predictions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/pasquale/git/ted-talk-topic-extraction/app/builtin/lftm/LFTM.jar\n",
      "/Users/pasquale/git/ted-talk-topic-extraction/app/builtin/glove/glove.6B.50d.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import time\n",
    "import json \n",
    "import pandas as pd\n",
    "from app.builtin import *\n",
    "\n",
    "sys.path.append('../app')  # for let gsdmm to work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "models= {\n",
    "#     'ntm': NtmModel,\n",
    "    'lda': LdaModel,\n",
    "    'lftm': LftmModel,\n",
    "#     'gsdmm': GsdmmModel\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 20 NG Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "dataset = '../data/20ng.txt'\n",
    "labels = '../data/20ng_labels.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['rec.autos',\n",
       " 'comp.sys.mac.hardware',\n",
       " 'comp.sys.mac.hardware',\n",
       " 'comp.graphics',\n",
       " 'sci.space',\n",
       " 'talk.politics.guns',\n",
       " 'sci.med',\n",
       " 'comp.sys.ibm.pc.hardware',\n",
       " 'comp.os.ms-windows.misc',\n",
       " 'comp.sys.mac.hardware']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(labels, \"r\") as datafile:\n",
    "    true_topics = [line.rstrip() for line in datafile if line]\n",
    "true_topics[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0417 13:45:59.506932 4599958976 deprecation.py:506] From /usr/local/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ntm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0417 13:45:59.693647 4599958976 deprecation.py:323] From /usr/local/lib/python3.7/site-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "W0417 13:46:00.028012 4599958976 module_wrapper.py:139] From /usr/local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done in 1.9874699115753174\n",
      "lda\n",
      "Done in 0.5751392841339111\n",
      "lftm\n",
      "Done in 0.26327991485595703\n",
      "gsdmm\n",
      "Done in 0.38397884368896484\n"
     ]
    }
   ],
   "source": [
    "results = {}\n",
    "for name, model in models.items():\n",
    "    start = time.time()\n",
    "    m = model()\n",
    "    print(name)\n",
    "    results[name] = m.get_corpus_predictions()\n",
    "    dur = time.time() - start\n",
    "    print('Done in ' + str(dur))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true</th>\n",
       "      <th>ntm_topic_0</th>\n",
       "      <th>ntm_score_0</th>\n",
       "      <th>ntm_topic_1</th>\n",
       "      <th>ntm_score_1</th>\n",
       "      <th>ntm_topic_2</th>\n",
       "      <th>ntm_score_2</th>\n",
       "      <th>lda_topic_0</th>\n",
       "      <th>lda_score_0</th>\n",
       "      <th>lda_topic_1</th>\n",
       "      <th>...</th>\n",
       "      <th>lftm_topic_1</th>\n",
       "      <th>lftm_score_1</th>\n",
       "      <th>lftm_topic_2</th>\n",
       "      <th>lftm_score_2</th>\n",
       "      <th>gsdmm_topic_0</th>\n",
       "      <th>gsdmm_score_0</th>\n",
       "      <th>gsdmm_topic_1</th>\n",
       "      <th>gsdmm_score_1</th>\n",
       "      <th>gsdmm_topic_2</th>\n",
       "      <th>gsdmm_score_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rec.autos</td>\n",
       "      <td>11</td>\n",
       "      <td>0.227447</td>\n",
       "      <td>4</td>\n",
       "      <td>0.202119</td>\n",
       "      <td>5</td>\n",
       "      <td>0.163517</td>\n",
       "      <td>6</td>\n",
       "      <td>0.845002</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0.220588</td>\n",
       "      <td>16</td>\n",
       "      <td>0.073529</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999993</td>\n",
       "      <td>19</td>\n",
       "      <td>6.891301e-06</td>\n",
       "      <td>11</td>\n",
       "      <td>2.122357e-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>comp.sys.mac.hardware</td>\n",
       "      <td>14</td>\n",
       "      <td>0.249227</td>\n",
       "      <td>18</td>\n",
       "      <td>0.174287</td>\n",
       "      <td>0</td>\n",
       "      <td>0.165302</td>\n",
       "      <td>5</td>\n",
       "      <td>0.451678</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.146067</td>\n",
       "      <td>17</td>\n",
       "      <td>0.089888</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>19</td>\n",
       "      <td>1.552614e-14</td>\n",
       "      <td>4</td>\n",
       "      <td>5.241309e-31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>comp.sys.mac.hardware</td>\n",
       "      <td>7</td>\n",
       "      <td>0.333553</td>\n",
       "      <td>14</td>\n",
       "      <td>0.272124</td>\n",
       "      <td>5</td>\n",
       "      <td>0.116028</td>\n",
       "      <td>12</td>\n",
       "      <td>0.521352</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>0.160714</td>\n",
       "      <td>7</td>\n",
       "      <td>0.089286</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>comp.graphics</td>\n",
       "      <td>16</td>\n",
       "      <td>0.254609</td>\n",
       "      <td>0</td>\n",
       "      <td>0.196284</td>\n",
       "      <td>2</td>\n",
       "      <td>0.137768</td>\n",
       "      <td>5</td>\n",
       "      <td>0.602450</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>19</td>\n",
       "      <td>0.098592</td>\n",
       "      <td>6</td>\n",
       "      <td>0.084507</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>19</td>\n",
       "      <td>7.681433e-09</td>\n",
       "      <td>4</td>\n",
       "      <td>4.168175e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sci.space</td>\n",
       "      <td>5</td>\n",
       "      <td>0.292701</td>\n",
       "      <td>0</td>\n",
       "      <td>0.186552</td>\n",
       "      <td>10</td>\n",
       "      <td>0.141531</td>\n",
       "      <td>12</td>\n",
       "      <td>0.592231</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.134021</td>\n",
       "      <td>2</td>\n",
       "      <td>0.134021</td>\n",
       "      <td>19</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>7.832201e-09</td>\n",
       "      <td>1</td>\n",
       "      <td>6.731073e-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11309</th>\n",
       "      <td>sci.med</td>\n",
       "      <td>7</td>\n",
       "      <td>0.601212</td>\n",
       "      <td>8</td>\n",
       "      <td>0.236097</td>\n",
       "      <td>2</td>\n",
       "      <td>0.131402</td>\n",
       "      <td>14</td>\n",
       "      <td>0.730352</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>0.085714</td>\n",
       "      <td>3</td>\n",
       "      <td>0.080000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11310</th>\n",
       "      <td>comp.sys.mac.hardware</td>\n",
       "      <td>15</td>\n",
       "      <td>0.339457</td>\n",
       "      <td>2</td>\n",
       "      <td>0.160176</td>\n",
       "      <td>19</td>\n",
       "      <td>0.155714</td>\n",
       "      <td>5</td>\n",
       "      <td>0.719951</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>17</td>\n",
       "      <td>0.126582</td>\n",
       "      <td>13</td>\n",
       "      <td>0.101266</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>7.849075e-19</td>\n",
       "      <td>19</td>\n",
       "      <td>1.750207e-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11311</th>\n",
       "      <td>comp.sys.ibm.pc.hardware</td>\n",
       "      <td>9</td>\n",
       "      <td>0.211234</td>\n",
       "      <td>14</td>\n",
       "      <td>0.181800</td>\n",
       "      <td>12</td>\n",
       "      <td>0.162045</td>\n",
       "      <td>5</td>\n",
       "      <td>0.511405</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.202532</td>\n",
       "      <td>3</td>\n",
       "      <td>0.088608</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>19</td>\n",
       "      <td>8.755647e-13</td>\n",
       "      <td>4</td>\n",
       "      <td>6.851880e-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11312</th>\n",
       "      <td>comp.graphics</td>\n",
       "      <td>4</td>\n",
       "      <td>0.256770</td>\n",
       "      <td>7</td>\n",
       "      <td>0.198837</td>\n",
       "      <td>16</td>\n",
       "      <td>0.172548</td>\n",
       "      <td>2</td>\n",
       "      <td>0.748150</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>19</td>\n",
       "      <td>0.092593</td>\n",
       "      <td>2</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>4</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>19</td>\n",
       "      <td>3.562953e-11</td>\n",
       "      <td>1</td>\n",
       "      <td>1.542093e-31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11313</th>\n",
       "      <td>rec.motorcycles</td>\n",
       "      <td>8</td>\n",
       "      <td>0.574201</td>\n",
       "      <td>0</td>\n",
       "      <td>0.424364</td>\n",
       "      <td>5</td>\n",
       "      <td>0.001435</td>\n",
       "      <td>6</td>\n",
       "      <td>0.885666</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0.115385</td>\n",
       "      <td>17</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>19</td>\n",
       "      <td>5.806591e-09</td>\n",
       "      <td>4</td>\n",
       "      <td>1.059958e-19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11314 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           true  ntm_topic_0  ntm_score_0  ntm_topic_1  \\\n",
       "0                     rec.autos           11     0.227447            4   \n",
       "1         comp.sys.mac.hardware           14     0.249227           18   \n",
       "2         comp.sys.mac.hardware            7     0.333553           14   \n",
       "3                 comp.graphics           16     0.254609            0   \n",
       "4                     sci.space            5     0.292701            0   \n",
       "...                         ...          ...          ...          ...   \n",
       "11309                   sci.med            7     0.601212            8   \n",
       "11310     comp.sys.mac.hardware           15     0.339457            2   \n",
       "11311  comp.sys.ibm.pc.hardware            9     0.211234           14   \n",
       "11312             comp.graphics            4     0.256770            7   \n",
       "11313           rec.motorcycles            8     0.574201            0   \n",
       "\n",
       "       ntm_score_1  ntm_topic_2  ntm_score_2  lda_topic_0  lda_score_0  \\\n",
       "0         0.202119            5     0.163517            6     0.845002   \n",
       "1         0.174287            0     0.165302            5     0.451678   \n",
       "2         0.272124            5     0.116028           12     0.521352   \n",
       "3         0.196284            2     0.137768            5     0.602450   \n",
       "4         0.186552           10     0.141531           12     0.592231   \n",
       "...            ...          ...          ...          ...          ...   \n",
       "11309     0.236097            2     0.131402           14     0.730352   \n",
       "11310     0.160176           19     0.155714            5     0.719951   \n",
       "11311     0.181800           12     0.162045            5     0.511405   \n",
       "11312     0.198837           16     0.172548            2     0.748150   \n",
       "11313     0.424364            5     0.001435            6     0.885666   \n",
       "\n",
       "       lda_topic_1  ...  lftm_topic_1  lftm_score_1  lftm_topic_2  \\\n",
       "0               12  ...             3      0.220588            16   \n",
       "1               12  ...             8      0.146067            17   \n",
       "2                5  ...            10      0.160714             7   \n",
       "3               12  ...            19      0.098592             6   \n",
       "4                9  ...             1      0.134021             2   \n",
       "...            ...  ...           ...           ...           ...   \n",
       "11309           12  ...            10      0.085714             3   \n",
       "11310           16  ...            17      0.126582            13   \n",
       "11311           16  ...             8      0.202532             3   \n",
       "11312           12  ...            19      0.092593             2   \n",
       "11313           15  ...             3      0.115385            17   \n",
       "\n",
       "       lftm_score_2  gsdmm_topic_0  gsdmm_score_0  gsdmm_topic_1  \\\n",
       "0          0.073529              1       0.999993             19   \n",
       "1          0.089888              1       1.000000             19   \n",
       "2          0.089286              0       0.000000              1   \n",
       "3          0.084507              1       1.000000             19   \n",
       "4          0.134021             19       1.000000              4   \n",
       "...             ...            ...            ...            ...   \n",
       "11309      0.080000              0       0.000000              1   \n",
       "11310      0.101266              1       1.000000              4   \n",
       "11311      0.088608              1       1.000000             19   \n",
       "11312      0.083333              4       1.000000             19   \n",
       "11313      0.076923              1       1.000000             19   \n",
       "\n",
       "       gsdmm_score_1  gsdmm_topic_2  gsdmm_score_2  \n",
       "0       6.891301e-06             11   2.122357e-17  \n",
       "1       1.552614e-14              4   5.241309e-31  \n",
       "2       0.000000e+00              2   0.000000e+00  \n",
       "3       7.681433e-09              4   4.168175e-10  \n",
       "4       7.832201e-09              1   6.731073e-11  \n",
       "...              ...            ...            ...  \n",
       "11309   0.000000e+00              2   0.000000e+00  \n",
       "11310   7.849075e-19             19   1.750207e-19  \n",
       "11311   8.755647e-13              4   6.851880e-28  \n",
       "11312   3.562953e-11              1   1.542093e-31  \n",
       "11313   5.806591e-09              4   1.059958e-19  \n",
       "\n",
       "[11314 rows x 25 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "df['true'] = true_topics\n",
    "for name, model in models.items():\n",
    "    r = results[name]\n",
    "    for i in range(0,3):\n",
    "        key, value = zip(*[x[i] if len(x) > i else (-1, -1) for x in r])\n",
    "        df['%s_topic_%d'%(name,i)] = key\n",
    "        df['%s_score_%d'%(name,i)] = value\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.index.name = 'id'\n",
    "df.to_csv('./out/predictions_20ng.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, model in models.items():\n",
    "    m = model()\n",
    "    with open('./out/'+name+'_topics_20ng.json', 'w') as file:\n",
    "        json.dump(m.topics(), file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AFP dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = '../data/afp.txt'\n",
    "labels = '../data/afp_labels.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[',,',\n",
       " '13000000,13007000 13006000 13008000,',\n",
       " '11000000,11014000 11001000 11002000 11006000,',\n",
       " '11000000,11014000 11001000 11002000 11006000,',\n",
       " '11000000,11014000 11001000 11002000 11006000,',\n",
       " '11000000,11014000 11001000 11002000 11006000,',\n",
       " '15000000,,',\n",
       " '11000000,11014000 11001000 11002000 11006000,',\n",
       " '02000000,02003000 02001000 16001000,02003001',\n",
       " '15000000,15054000,']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(labels, \"r\") as datafile:\n",
    "    true_topics = [line.rstrip() for line in datafile if line]\n",
    "true_topics[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('',\n",
       " '13000000',\n",
       " '11000000',\n",
       " '11000000',\n",
       " '11000000',\n",
       " '11000000',\n",
       " '15000000',\n",
       " '11000000',\n",
       " '02000000',\n",
       " '15000000')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "true1, true2, true3 = zip(*[line.split(',') for line in true_topics])\n",
    "true1[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0417 13:45:03.727105 4663692736 deprecation.py:506] From /usr/local/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ntm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0417 13:45:03.899070 4663692736 deprecation.py:323] From /usr/local/lib/python3.7/site-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "W0417 13:45:04.289768 4663692736 module_wrapper.py:139] From /usr/local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "/Users/pasquale/git/ted-talk-topic-extraction/app/builtin/doc2topic/models.py:18: RuntimeWarning: invalid value encountered in true_divide\n",
      "  L1normalize = (lambda x: x/L1(x))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done in 5.400324106216431\n",
      "lda\n",
      "Done in 4.530890941619873\n",
      "lftm\n",
      "Done in 3.830026865005493\n",
      "gsdmm\n",
      "Done in 3.0285561084747314\n"
     ]
    }
   ],
   "source": [
    "results = {}\n",
    "for name, model in models.items():\n",
    "    start = time.time()\n",
    "    m = model()\n",
    "    print(name)\n",
    "    results[name] = m.get_corpus_predictions()\n",
    "    dur = time.time() - start\n",
    "    print('Done in ' + str(dur))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true</th>\n",
       "      <th>true2</th>\n",
       "      <th>true3</th>\n",
       "      <th>ntm_topic_0</th>\n",
       "      <th>ntm_score_0</th>\n",
       "      <th>ntm_topic_1</th>\n",
       "      <th>ntm_score_1</th>\n",
       "      <th>ntm_topic_2</th>\n",
       "      <th>ntm_score_2</th>\n",
       "      <th>lda_topic_0</th>\n",
       "      <th>...</th>\n",
       "      <th>lftm_topic_1</th>\n",
       "      <th>lftm_score_1</th>\n",
       "      <th>lftm_topic_2</th>\n",
       "      <th>lftm_score_2</th>\n",
       "      <th>gsdmm_topic_0</th>\n",
       "      <th>gsdmm_score_0</th>\n",
       "      <th>gsdmm_topic_1</th>\n",
       "      <th>gsdmm_score_1</th>\n",
       "      <th>gsdmm_topic_2</th>\n",
       "      <th>gsdmm_score_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.342370</td>\n",
       "      <td>1</td>\n",
       "      <td>0.323482</td>\n",
       "      <td>16</td>\n",
       "      <td>0.244672</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.006803</td>\n",
       "      <td>2</td>\n",
       "      <td>0.006803</td>\n",
       "      <td>15</td>\n",
       "      <td>0.999825</td>\n",
       "      <td>16</td>\n",
       "      <td>0.000175</td>\n",
       "      <td>5</td>\n",
       "      <td>2.455619e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13000000</td>\n",
       "      <td>13007000 13006000 13008000</td>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.296621</td>\n",
       "      <td>9</td>\n",
       "      <td>0.129710</td>\n",
       "      <td>1</td>\n",
       "      <td>0.121188</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>0.251570</td>\n",
       "      <td>3</td>\n",
       "      <td>0.055781</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11000000</td>\n",
       "      <td>11014000 11001000 11002000 11006000</td>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.694453</td>\n",
       "      <td>7</td>\n",
       "      <td>0.292373</td>\n",
       "      <td>0</td>\n",
       "      <td>0.007895</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>0.179487</td>\n",
       "      <td>2</td>\n",
       "      <td>0.094017</td>\n",
       "      <td>10</td>\n",
       "      <td>0.999995</td>\n",
       "      <td>16</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>5</td>\n",
       "      <td>1.009442e-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11000000</td>\n",
       "      <td>11014000 11001000 11002000 11006000</td>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.420780</td>\n",
       "      <td>7</td>\n",
       "      <td>0.325828</td>\n",
       "      <td>8</td>\n",
       "      <td>0.231620</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0.196262</td>\n",
       "      <td>15</td>\n",
       "      <td>0.196262</td>\n",
       "      <td>10</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>16</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>15</td>\n",
       "      <td>2.206482e-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11000000</td>\n",
       "      <td>11014000 11001000 11002000 11006000</td>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.413961</td>\n",
       "      <td>13</td>\n",
       "      <td>0.412301</td>\n",
       "      <td>16</td>\n",
       "      <td>0.135707</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>0.165354</td>\n",
       "      <td>10</td>\n",
       "      <td>0.086614</td>\n",
       "      <td>10</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>16</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>15</td>\n",
       "      <td>7.401703e-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125511</th>\n",
       "      <td>16000000</td>\n",
       "      <td>16002000 16001000</td>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.410465</td>\n",
       "      <td>2</td>\n",
       "      <td>0.200190</td>\n",
       "      <td>11</td>\n",
       "      <td>0.109299</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>0.093545</td>\n",
       "      <td>3</td>\n",
       "      <td>0.055575</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125512</th>\n",
       "      <td>01000000 08000000</td>\n",
       "      <td>01016000 08006000 01005000 01007000 01026000</td>\n",
       "      <td>08003002 08003003</td>\n",
       "      <td>10</td>\n",
       "      <td>0.280207</td>\n",
       "      <td>9</td>\n",
       "      <td>0.143741</td>\n",
       "      <td>11</td>\n",
       "      <td>0.138465</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.048621</td>\n",
       "      <td>3</td>\n",
       "      <td>0.045778</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125513</th>\n",
       "      <td>01000000 02000000</td>\n",
       "      <td>01026000</td>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.228517</td>\n",
       "      <td>8</td>\n",
       "      <td>0.194645</td>\n",
       "      <td>2</td>\n",
       "      <td>0.189475</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>0.071784</td>\n",
       "      <td>9</td>\n",
       "      <td>0.036247</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125514</th>\n",
       "      <td>14000000 08000000 11000000</td>\n",
       "      <td>14023000 08003000 12006000</td>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.275202</td>\n",
       "      <td>3</td>\n",
       "      <td>0.211507</td>\n",
       "      <td>12</td>\n",
       "      <td>0.205389</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>0.250592</td>\n",
       "      <td>10</td>\n",
       "      <td>0.169428</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125515</th>\n",
       "      <td>11000000 16000000</td>\n",
       "      <td>11014000 11002000</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.204295</td>\n",
       "      <td>11</td>\n",
       "      <td>0.203445</td>\n",
       "      <td>14</td>\n",
       "      <td>0.161406</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0.057478</td>\n",
       "      <td>9</td>\n",
       "      <td>0.051759</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125516 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              true  \\\n",
       "0                                    \n",
       "1                         13000000   \n",
       "2                         11000000   \n",
       "3                         11000000   \n",
       "4                         11000000   \n",
       "...                            ...   \n",
       "125511                    16000000   \n",
       "125512           01000000 08000000   \n",
       "125513           01000000 02000000   \n",
       "125514  14000000 08000000 11000000   \n",
       "125515           11000000 16000000   \n",
       "\n",
       "                                               true2              true3  \\\n",
       "0                                                                         \n",
       "1                         13007000 13006000 13008000                      \n",
       "2                11014000 11001000 11002000 11006000                      \n",
       "3                11014000 11001000 11002000 11006000                      \n",
       "4                11014000 11001000 11002000 11006000                      \n",
       "...                                              ...                ...   \n",
       "125511                             16002000 16001000                      \n",
       "125512  01016000 08006000 01005000 01007000 01026000  08003002 08003003   \n",
       "125513                                      01026000                      \n",
       "125514                    14023000 08003000 12006000                      \n",
       "125515                             11014000 11002000                      \n",
       "\n",
       "        ntm_topic_0  ntm_score_0  ntm_topic_1  ntm_score_1  ntm_topic_2  \\\n",
       "0                 9     0.342370            1     0.323482           16   \n",
       "1                 5     0.296621            9     0.129710            1   \n",
       "2                13     0.694453            7     0.292373            0   \n",
       "3                13     0.420780            7     0.325828            8   \n",
       "4                 7     0.413961           13     0.412301           16   \n",
       "...             ...          ...          ...          ...          ...   \n",
       "125511            9     0.410465            2     0.200190           11   \n",
       "125512           10     0.280207            9     0.143741           11   \n",
       "125513           12     0.228517            8     0.194645            2   \n",
       "125514           13     0.275202            3     0.211507           12   \n",
       "125515            1     0.204295           11     0.203445           14   \n",
       "\n",
       "        ntm_score_2  lda_topic_0  ...  lftm_topic_1  lftm_score_1  \\\n",
       "0          0.244672            0  ...             1      0.006803   \n",
       "1          0.121188           12  ...             7      0.251570   \n",
       "2          0.007895            5  ...            15      0.179487   \n",
       "3          0.231620           14  ...             3      0.196262   \n",
       "4          0.135707           12  ...            15      0.165354   \n",
       "...             ...          ...  ...           ...           ...   \n",
       "125511     0.109299           14  ...            12      0.093545   \n",
       "125512     0.138465           12  ...             8      0.048621   \n",
       "125513     0.189475           13  ...             5      0.071784   \n",
       "125514     0.205389            0  ...            12      0.250592   \n",
       "125515     0.161406           13  ...             3      0.057478   \n",
       "\n",
       "        lftm_topic_2  lftm_score_2  gsdmm_topic_0  gsdmm_score_0  \\\n",
       "0                  2      0.006803             15       0.999825   \n",
       "1                  3      0.055781              0       0.000000   \n",
       "2                  2      0.094017             10       0.999995   \n",
       "3                 15      0.196262             10       0.999999   \n",
       "4                 10      0.086614             10       0.999998   \n",
       "...              ...           ...            ...            ...   \n",
       "125511             3      0.055575              0       0.000000   \n",
       "125512             3      0.045778              0       0.000000   \n",
       "125513             9      0.036247              0       0.000000   \n",
       "125514            10      0.169428              0       0.000000   \n",
       "125515             9      0.051759              0       0.000000   \n",
       "\n",
       "        gsdmm_topic_1  gsdmm_score_1  gsdmm_topic_2  gsdmm_score_2  \n",
       "0                  16       0.000175              5   2.455619e-09  \n",
       "1                   1       0.000000              2   0.000000e+00  \n",
       "2                  16       0.000005              5   1.009442e-12  \n",
       "3                  16       0.000001             15   2.206482e-11  \n",
       "4                  16       0.000002             15   7.401703e-11  \n",
       "...               ...            ...            ...            ...  \n",
       "125511              1       0.000000              2   0.000000e+00  \n",
       "125512              1       0.000000              2   0.000000e+00  \n",
       "125513              1       0.000000              2   0.000000e+00  \n",
       "125514              1       0.000000              2   0.000000e+00  \n",
       "125515              1       0.000000              2   0.000000e+00  \n",
       "\n",
       "[125516 rows x 27 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "df['true'] = true1\n",
    "df['true2'] = true2\n",
    "df['true3'] = true3\n",
    "for name, model in models.items():\n",
    "    r = results[name]\n",
    "    for i in range(0,3):\n",
    "        key, value = zip(*[x[i] if len(x) > i else (-1, -1) for x in r])\n",
    "        df['%s_topic_%d'%(name,i)] = key\n",
    "        df['%s_score_%d'%(name,i)] = value\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.index.name = 'id'\n",
    "df.to_csv('./out/predictions_afp.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, model in models.items():\n",
    "    m = model()\n",
    "    with open('./out/'+name+'_topics_afp.json', 'w') as file:\n",
    "        json.dump(m.topics(), file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AFP mono label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = '../data/afp_mono.txt'\n",
    "labels = '../data/afp_mono_label.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(labels, \"r\") as datafile:\n",
    "    true_topics = [line.rstrip() for line in datafile if line]\n",
    "true_topics[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda\n",
      "Done in 2.0214741230010986\n",
      "lftm\n",
      "Done in 1.5935988426208496\n"
     ]
    }
   ],
   "source": [
    "results = {}\n",
    "for name, model in models.items():\n",
    "    start = time.time()\n",
    "    m = model()\n",
    "    print(name)\n",
    "    results[name] = m.get_corpus_predictions()\n",
    "    dur = time.time() - start\n",
    "    print('Done in ' + str(dur))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true</th>\n",
       "      <th>lda_topic_0</th>\n",
       "      <th>lda_score_0</th>\n",
       "      <th>lda_topic_1</th>\n",
       "      <th>lda_score_1</th>\n",
       "      <th>lda_topic_2</th>\n",
       "      <th>lda_score_2</th>\n",
       "      <th>lftm_topic_0</th>\n",
       "      <th>lftm_score_0</th>\n",
       "      <th>lftm_topic_1</th>\n",
       "      <th>lftm_score_1</th>\n",
       "      <th>lftm_topic_2</th>\n",
       "      <th>lftm_score_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.959057</td>\n",
       "      <td>3</td>\n",
       "      <td>0.039865</td>\n",
       "      <td>11</td>\n",
       "      <td>0.000160</td>\n",
       "      <td>1</td>\n",
       "      <td>0.455251</td>\n",
       "      <td>12</td>\n",
       "      <td>0.451553</td>\n",
       "      <td>8</td>\n",
       "      <td>0.048447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.980001</td>\n",
       "      <td>5</td>\n",
       "      <td>0.019294</td>\n",
       "      <td>11</td>\n",
       "      <td>0.000099</td>\n",
       "      <td>12</td>\n",
       "      <td>0.375977</td>\n",
       "      <td>8</td>\n",
       "      <td>0.336881</td>\n",
       "      <td>1</td>\n",
       "      <td>0.158775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.963631</td>\n",
       "      <td>11</td>\n",
       "      <td>0.004935</td>\n",
       "      <td>2</td>\n",
       "      <td>0.004745</td>\n",
       "      <td>12</td>\n",
       "      <td>0.490385</td>\n",
       "      <td>6</td>\n",
       "      <td>0.201923</td>\n",
       "      <td>7</td>\n",
       "      <td>0.201923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999257</td>\n",
       "      <td>11</td>\n",
       "      <td>0.000101</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000097</td>\n",
       "      <td>8</td>\n",
       "      <td>0.383184</td>\n",
       "      <td>12</td>\n",
       "      <td>0.351091</td>\n",
       "      <td>1</td>\n",
       "      <td>0.220582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.968004</td>\n",
       "      <td>11</td>\n",
       "      <td>0.004342</td>\n",
       "      <td>2</td>\n",
       "      <td>0.004175</td>\n",
       "      <td>1</td>\n",
       "      <td>0.436170</td>\n",
       "      <td>12</td>\n",
       "      <td>0.329787</td>\n",
       "      <td>5</td>\n",
       "      <td>0.117021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69049</th>\n",
       "      <td>14000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.906607</td>\n",
       "      <td>12</td>\n",
       "      <td>0.092759</td>\n",
       "      <td>11</td>\n",
       "      <td>0.000093</td>\n",
       "      <td>1</td>\n",
       "      <td>0.818079</td>\n",
       "      <td>9</td>\n",
       "      <td>0.068192</td>\n",
       "      <td>7</td>\n",
       "      <td>0.063661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69050</th>\n",
       "      <td>14000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.838466</td>\n",
       "      <td>7</td>\n",
       "      <td>0.158675</td>\n",
       "      <td>11</td>\n",
       "      <td>0.000409</td>\n",
       "      <td>3</td>\n",
       "      <td>0.379482</td>\n",
       "      <td>1</td>\n",
       "      <td>0.319721</td>\n",
       "      <td>4</td>\n",
       "      <td>0.220120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69051</th>\n",
       "      <td>14000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.797622</td>\n",
       "      <td>7</td>\n",
       "      <td>0.199418</td>\n",
       "      <td>11</td>\n",
       "      <td>0.000423</td>\n",
       "      <td>1</td>\n",
       "      <td>0.606776</td>\n",
       "      <td>3</td>\n",
       "      <td>0.237166</td>\n",
       "      <td>4</td>\n",
       "      <td>0.113963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69052</th>\n",
       "      <td>14000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.672036</td>\n",
       "      <td>4</td>\n",
       "      <td>0.198470</td>\n",
       "      <td>12</td>\n",
       "      <td>0.115278</td>\n",
       "      <td>3</td>\n",
       "      <td>0.719838</td>\n",
       "      <td>2</td>\n",
       "      <td>0.162993</td>\n",
       "      <td>13</td>\n",
       "      <td>0.058585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69053</th>\n",
       "      <td>14000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.811956</td>\n",
       "      <td>2</td>\n",
       "      <td>0.173200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.013437</td>\n",
       "      <td>1</td>\n",
       "      <td>0.516628</td>\n",
       "      <td>13</td>\n",
       "      <td>0.229931</td>\n",
       "      <td>5</td>\n",
       "      <td>0.092317</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>69054 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           true  lda_topic_0  lda_score_0  lda_topic_1  lda_score_1  \\\n",
       "0      13000000            1     0.959057            3     0.039865   \n",
       "1      13000000            1     0.980001            5     0.019294   \n",
       "2      13000000            1     0.963631           11     0.004935   \n",
       "3      13000000            1     0.999257           11     0.000101   \n",
       "4      13000000            1     0.968004           11     0.004342   \n",
       "...         ...          ...          ...          ...          ...   \n",
       "69049  14000000            1     0.906607           12     0.092759   \n",
       "69050  14000000            1     0.838466            7     0.158675   \n",
       "69051  14000000            1     0.797622            7     0.199418   \n",
       "69052  14000000            1     0.672036            4     0.198470   \n",
       "69053  14000000            1     0.811956            2     0.173200   \n",
       "\n",
       "       lda_topic_2  lda_score_2  lftm_topic_0  lftm_score_0  lftm_topic_1  \\\n",
       "0               11     0.000160             1      0.455251            12   \n",
       "1               11     0.000099            12      0.375977             8   \n",
       "2                2     0.004745            12      0.490385             6   \n",
       "3                2     0.000097             8      0.383184            12   \n",
       "4                2     0.004175             1      0.436170            12   \n",
       "...            ...          ...           ...           ...           ...   \n",
       "69049           11     0.000093             1      0.818079             9   \n",
       "69050           11     0.000409             3      0.379482             1   \n",
       "69051           11     0.000423             1      0.606776             3   \n",
       "69052           12     0.115278             3      0.719838             2   \n",
       "69053           12     0.013437             1      0.516628            13   \n",
       "\n",
       "       lftm_score_1  lftm_topic_2  lftm_score_2  \n",
       "0          0.451553             8      0.048447  \n",
       "1          0.336881             1      0.158775  \n",
       "2          0.201923             7      0.201923  \n",
       "3          0.351091             1      0.220582  \n",
       "4          0.329787             5      0.117021  \n",
       "...             ...           ...           ...  \n",
       "69049      0.068192             7      0.063661  \n",
       "69050      0.319721             4      0.220120  \n",
       "69051      0.237166             4      0.113963  \n",
       "69052      0.162993            13      0.058585  \n",
       "69053      0.229931             5      0.092317  \n",
       "\n",
       "[69054 rows x 13 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "df['true'] = true_topics\n",
    "for name, model in models.items():\n",
    "    r = results[name]\n",
    "    for i in range(0,3):\n",
    "        key, value = zip(*[x[i] if len(x) > i else (-1, -1) for x in r])\n",
    "        df['%s_topic_%d'%(name,i)] = key\n",
    "        df['%s_score_%d'%(name,i)] = value\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.index.name = 'id'\n",
    "df.to_csv('./out/predictions_afp_mono.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, model in models.items():\n",
    "    m = model()\n",
    "    with open('./out/'+name+'_topics_afp_mono.json', 'w') as file:\n",
    "        json.dump(m.topics(), file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AFP balanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = '../data/afp_balanced.txt'\n",
    "labels = '../data/afp_balanced_label.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000',\n",
       " '13000000']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(labels, \"r\") as datafile:\n",
    "    true_topics = [line.rstrip() for line in datafile if line]\n",
    "true_topics[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda\n",
      "Done in 0.08881211280822754\n",
      "lftm\n",
      "Done in 0.03905367851257324\n"
     ]
    }
   ],
   "source": [
    "results = {}\n",
    "for name, model in models.items():\n",
    "    start = time.time()\n",
    "    m = model()\n",
    "    print(name)\n",
    "    results[name] = m.get_corpus_predictions()\n",
    "    dur = time.time() - start\n",
    "    print('Done in ' + str(dur))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true</th>\n",
       "      <th>lda_topic_0</th>\n",
       "      <th>lda_score_0</th>\n",
       "      <th>lda_topic_1</th>\n",
       "      <th>lda_score_1</th>\n",
       "      <th>lda_topic_2</th>\n",
       "      <th>lda_score_2</th>\n",
       "      <th>lftm_topic_0</th>\n",
       "      <th>lftm_score_0</th>\n",
       "      <th>lftm_topic_1</th>\n",
       "      <th>lftm_score_1</th>\n",
       "      <th>lftm_topic_2</th>\n",
       "      <th>lftm_score_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13000000</td>\n",
       "      <td>9</td>\n",
       "      <td>0.784033</td>\n",
       "      <td>12</td>\n",
       "      <td>0.129114</td>\n",
       "      <td>4</td>\n",
       "      <td>0.040632</td>\n",
       "      <td>2</td>\n",
       "      <td>0.764997</td>\n",
       "      <td>6</td>\n",
       "      <td>0.099209</td>\n",
       "      <td>8</td>\n",
       "      <td>0.053065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13000000</td>\n",
       "      <td>13</td>\n",
       "      <td>0.539797</td>\n",
       "      <td>6</td>\n",
       "      <td>0.253352</td>\n",
       "      <td>7</td>\n",
       "      <td>0.156515</td>\n",
       "      <td>8</td>\n",
       "      <td>0.509108</td>\n",
       "      <td>3</td>\n",
       "      <td>0.191897</td>\n",
       "      <td>6</td>\n",
       "      <td>0.103957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13000000</td>\n",
       "      <td>9</td>\n",
       "      <td>0.983287</td>\n",
       "      <td>10</td>\n",
       "      <td>0.002093</td>\n",
       "      <td>12</td>\n",
       "      <td>0.002001</td>\n",
       "      <td>2</td>\n",
       "      <td>0.639151</td>\n",
       "      <td>9</td>\n",
       "      <td>0.238208</td>\n",
       "      <td>7</td>\n",
       "      <td>0.049528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13000000</td>\n",
       "      <td>9</td>\n",
       "      <td>0.581149</td>\n",
       "      <td>8</td>\n",
       "      <td>0.158394</td>\n",
       "      <td>13</td>\n",
       "      <td>0.106626</td>\n",
       "      <td>2</td>\n",
       "      <td>0.666230</td>\n",
       "      <td>6</td>\n",
       "      <td>0.152334</td>\n",
       "      <td>1</td>\n",
       "      <td>0.113005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13000000</td>\n",
       "      <td>10</td>\n",
       "      <td>0.334313</td>\n",
       "      <td>4</td>\n",
       "      <td>0.323300</td>\n",
       "      <td>9</td>\n",
       "      <td>0.194010</td>\n",
       "      <td>8</td>\n",
       "      <td>0.659892</td>\n",
       "      <td>2</td>\n",
       "      <td>0.208220</td>\n",
       "      <td>6</td>\n",
       "      <td>0.108853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2221</th>\n",
       "      <td>14000000</td>\n",
       "      <td>10</td>\n",
       "      <td>0.985187</td>\n",
       "      <td>12</td>\n",
       "      <td>0.013382</td>\n",
       "      <td>6</td>\n",
       "      <td>0.000206</td>\n",
       "      <td>10</td>\n",
       "      <td>0.496392</td>\n",
       "      <td>6</td>\n",
       "      <td>0.146820</td>\n",
       "      <td>4</td>\n",
       "      <td>0.137799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2222</th>\n",
       "      <td>14000000</td>\n",
       "      <td>10</td>\n",
       "      <td>0.659714</td>\n",
       "      <td>12</td>\n",
       "      <td>0.231240</td>\n",
       "      <td>11</td>\n",
       "      <td>0.092632</td>\n",
       "      <td>12</td>\n",
       "      <td>0.476218</td>\n",
       "      <td>3</td>\n",
       "      <td>0.220998</td>\n",
       "      <td>6</td>\n",
       "      <td>0.197796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2223</th>\n",
       "      <td>14000000</td>\n",
       "      <td>10</td>\n",
       "      <td>0.474810</td>\n",
       "      <td>6</td>\n",
       "      <td>0.380798</td>\n",
       "      <td>0</td>\n",
       "      <td>0.096766</td>\n",
       "      <td>1</td>\n",
       "      <td>0.411290</td>\n",
       "      <td>3</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>11</td>\n",
       "      <td>0.169355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2224</th>\n",
       "      <td>14000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.825013</td>\n",
       "      <td>3</td>\n",
       "      <td>0.173205</td>\n",
       "      <td>10</td>\n",
       "      <td>0.000246</td>\n",
       "      <td>10</td>\n",
       "      <td>0.613176</td>\n",
       "      <td>6</td>\n",
       "      <td>0.152268</td>\n",
       "      <td>5</td>\n",
       "      <td>0.142616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2225</th>\n",
       "      <td>14000000</td>\n",
       "      <td>10</td>\n",
       "      <td>0.668143</td>\n",
       "      <td>8</td>\n",
       "      <td>0.270604</td>\n",
       "      <td>11</td>\n",
       "      <td>0.037376</td>\n",
       "      <td>4</td>\n",
       "      <td>0.407116</td>\n",
       "      <td>10</td>\n",
       "      <td>0.273105</td>\n",
       "      <td>3</td>\n",
       "      <td>0.162200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2226 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          true  lda_topic_0  lda_score_0  lda_topic_1  lda_score_1  \\\n",
       "0     13000000            9     0.784033           12     0.129114   \n",
       "1     13000000           13     0.539797            6     0.253352   \n",
       "2     13000000            9     0.983287           10     0.002093   \n",
       "3     13000000            9     0.581149            8     0.158394   \n",
       "4     13000000           10     0.334313            4     0.323300   \n",
       "...        ...          ...          ...          ...          ...   \n",
       "2221  14000000           10     0.985187           12     0.013382   \n",
       "2222  14000000           10     0.659714           12     0.231240   \n",
       "2223  14000000           10     0.474810            6     0.380798   \n",
       "2224  14000000            4     0.825013            3     0.173205   \n",
       "2225  14000000           10     0.668143            8     0.270604   \n",
       "\n",
       "      lda_topic_2  lda_score_2  lftm_topic_0  lftm_score_0  lftm_topic_1  \\\n",
       "0               4     0.040632             2      0.764997             6   \n",
       "1               7     0.156515             8      0.509108             3   \n",
       "2              12     0.002001             2      0.639151             9   \n",
       "3              13     0.106626             2      0.666230             6   \n",
       "4               9     0.194010             8      0.659892             2   \n",
       "...           ...          ...           ...           ...           ...   \n",
       "2221            6     0.000206            10      0.496392             6   \n",
       "2222           11     0.092632            12      0.476218             3   \n",
       "2223            0     0.096766             1      0.411290             3   \n",
       "2224           10     0.000246            10      0.613176             6   \n",
       "2225           11     0.037376             4      0.407116            10   \n",
       "\n",
       "      lftm_score_1  lftm_topic_2  lftm_score_2  \n",
       "0         0.099209             8      0.053065  \n",
       "1         0.191897             6      0.103957  \n",
       "2         0.238208             7      0.049528  \n",
       "3         0.152334             1      0.113005  \n",
       "4         0.208220             6      0.108853  \n",
       "...            ...           ...           ...  \n",
       "2221      0.146820             4      0.137799  \n",
       "2222      0.220998             6      0.197796  \n",
       "2223      0.250000            11      0.169355  \n",
       "2224      0.152268             5      0.142616  \n",
       "2225      0.273105             3      0.162200  \n",
       "\n",
       "[2226 rows x 13 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "df['true'] = true_topics\n",
    "for name, model in models.items():\n",
    "    r = results[name]\n",
    "    for i in range(0,3):\n",
    "        key, value = zip(*[x[i] if len(x) > i else (-1, -1) for x in r])\n",
    "        df['%s_topic_%d'%(name,i)] = key\n",
    "        df['%s_score_%d'%(name,i)] = value\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.index.name = 'id'\n",
    "df.to_csv('./out/predictions_afp_balanced.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, model in models.items():\n",
    "    m = model()\n",
    "    with open('./out/'+name+'_topics_afp_balanced.json', 'w') as file:\n",
    "        json.dump(m.topics(), file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}