project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os.path\n",
    "import pickle \n",
    "import numpy as np\n",
    "import tensorflow.keras.utils\n",
    "\n",
    "import time\n",
    "from tensorflow.keras.callbacks import TensorBoard\n",
    "from tensorflow.keras.callbacks import CSVLogger\n",
    "#from keras.callbacks import CSVLogger\n",
    "from tensorflow.keras.preprocessing.text import text_to_word_sequence\n",
    "from tensorflow.keras.preprocessing import sequence\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense,Flatten,LSTM,Conv1D,GlobalMaxPool1D,Dropout,Bidirectional\n",
    "from tensorflow.keras.layers import Embedding\n",
    "from tensorflow.keras import optimizers\n",
    "from tensorflow.keras.layers import Input\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.utils import plot_model\n",
    "from IPython.display import SVG\n",
    "from tensorflow.keras.utils import model_to_dot\n",
    "from tensorflow.keras.models import load_model\n",
    "from nltk.corpus import stopwords\n",
    "import operator\n",
    "import nltk\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_rows\n",
    "pd.set_option('display.max_colwidth', 1)\n",
    "from tensorflow.keras import backend as K\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_table('train.tsv', names = [\"id\", \"label\", \"statement\", \"subject\", \"speaker\", \"job\", \"state\", \"party\",\n",
    "                                            \"barely-true\", \"false\", \"half-true\", \"mostly-true\", \"pants-fire\", \"venue\"])\n",
    "\n",
    "val_data = pd.read_table('valid.tsv', names = [\"id\", \"label\", \"statement\", \"subject\", \"speaker\", \"job\", \"state\", \"party\",\n",
    "                                            \"barely-true\", \"false\", \"half-true\", \"mostly-true\", \"pants-fire\", \"venue\"])\n",
    "\n",
    "test_data = pd.read_table('test.tsv', names = [\"id\", \"label\", \"statement\", \"subject\", \"speaker\", \"job\", \"state\", \"party\",\n",
    "                                            \"barely-true\", \"false\", \"half-true\", \"mostly-true\", \"pants-fire\", \"venue\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10240, 14) (1284, 14) (1267, 14)\n",
      "['false' 'half-true' 'mostly-true' 'true' 'barely-true' 'pants-fire']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>statement</th>\n",
       "      <th>subject</th>\n",
       "      <th>speaker</th>\n",
       "      <th>job</th>\n",
       "      <th>state</th>\n",
       "      <th>party</th>\n",
       "      <th>barely-true</th>\n",
       "      <th>false</th>\n",
       "      <th>half-true</th>\n",
       "      <th>mostly-true</th>\n",
       "      <th>pants-fire</th>\n",
       "      <th>venue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2635.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Says the Annies List political group supports third-trimester abortions on demand.</td>\n",
       "      <td>abortion</td>\n",
       "      <td>dwayne-bohac</td>\n",
       "      <td>State representative</td>\n",
       "      <td>Texas</td>\n",
       "      <td>republican</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>a mailer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10540.json</td>\n",
       "      <td>half-true</td>\n",
       "      <td>When did the decline of coal start? It started when natural gas took off that started to begin in (President George W.) Bushs administration.</td>\n",
       "      <td>energy,history,job-accomplishments</td>\n",
       "      <td>scott-surovell</td>\n",
       "      <td>State delegate</td>\n",
       "      <td>Virginia</td>\n",
       "      <td>democrat</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>a floor speech.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>324.json</td>\n",
       "      <td>mostly-true</td>\n",
       "      <td>Hillary Clinton agrees with John McCain \"by voting to give George Bush the benefit of the doubt on Iran.\"</td>\n",
       "      <td>foreign-policy</td>\n",
       "      <td>barack-obama</td>\n",
       "      <td>President</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>democrat</td>\n",
       "      <td>70.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Denver</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1123.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Health care reform legislation is likely to mandate free sex change surgeries.</td>\n",
       "      <td>health-care</td>\n",
       "      <td>blog-posting</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>none</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>a news release</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9028.json</td>\n",
       "      <td>half-true</td>\n",
       "      <td>The economic turnaround started at the end of my term.</td>\n",
       "      <td>economy,jobs</td>\n",
       "      <td>charlie-crist</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Florida</td>\n",
       "      <td>democrat</td>\n",
       "      <td>15.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>an interview on CNN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id        label  \\\n",
       "0  2635.json   false         \n",
       "1  10540.json  half-true     \n",
       "2  324.json    mostly-true   \n",
       "3  1123.json   false         \n",
       "4  9028.json   half-true     \n",
       "\n",
       "                                                                                                                                       statement  \\\n",
       "0  Says the Annies List political group supports third-trimester abortions on demand.                                                              \n",
       "1  When did the decline of coal start? It started when natural gas took off that started to begin in (President George W.) Bushs administration.   \n",
       "2  Hillary Clinton agrees with John McCain \"by voting to give George Bush the benefit of the doubt on Iran.\"                                       \n",
       "3  Health care reform legislation is likely to mandate free sex change surgeries.                                                                  \n",
       "4  The economic turnaround started at the end of my term.                                                                                          \n",
       "\n",
       "                              subject         speaker                   job  \\\n",
       "0  abortion                            dwayne-bohac    State representative   \n",
       "1  energy,history,job-accomplishments  scott-surovell  State delegate         \n",
       "2  foreign-policy                      barack-obama    President              \n",
       "3  health-care                         blog-posting    NaN                    \n",
       "4  economy,jobs                        charlie-crist   NaN                    \n",
       "\n",
       "      state       party  barely-true  false  half-true  mostly-true  \\\n",
       "0  Texas     republican  0.0          1.0    0.0        0.0           \n",
       "1  Virginia  democrat    0.0          0.0    1.0        1.0           \n",
       "2  Illinois  democrat    70.0         71.0   160.0      163.0         \n",
       "3  NaN       none        7.0          19.0   3.0        5.0           \n",
       "4  Florida   democrat    15.0         9.0    20.0       19.0          \n",
       "\n",
       "   pants-fire                venue  \n",
       "0  0.0         a mailer             \n",
       "1  0.0         a floor speech.      \n",
       "2  9.0         Denver               \n",
       "3  44.0        a news release       \n",
       "4  2.0         an interview on CNN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print (train_data.shape, val_data.shape, test_data.shape)\n",
    "print (train_data.label.unique())\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 10240 entries, 0 to 10239\n",
      "Data columns (total 14 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   id           10240 non-null  object \n",
      " 1   label        10240 non-null  object \n",
      " 2   statement    10240 non-null  object \n",
      " 3   subject      10238 non-null  object \n",
      " 4   speaker      10238 non-null  object \n",
      " 5   job          7343 non-null   object \n",
      " 6   state        8032 non-null   object \n",
      " 7   party        10238 non-null  object \n",
      " 8   barely-true  10238 non-null  float64\n",
      " 9   false        10238 non-null  float64\n",
      " 10  half-true    10238 non-null  float64\n",
      " 11  mostly-true  10238 non-null  float64\n",
      " 12  pants-fire   10238 non-null  float64\n",
      " 13  venue        10138 non-null  object \n",
      "dtypes: float64(5), object(9)\n",
      "memory usage: 1.1+ MB\n",
      "None\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1284 entries, 0 to 1283\n",
      "Data columns (total 14 columns):\n",
      " #   Column       Non-Null Count  Dtype \n",
      "---  ------       --------------  ----- \n",
      " 0   id           1284 non-null   object\n",
      " 1   label        1284 non-null   object\n",
      " 2   statement    1284 non-null   object\n",
      " 3   subject      1284 non-null   object\n",
      " 4   speaker      1284 non-null   object\n",
      " 5   job          939 non-null    object\n",
      " 6   state        1005 non-null   object\n",
      " 7   party        1284 non-null   object\n",
      " 8   barely-true  1284 non-null   int64 \n",
      " 9   false        1284 non-null   int64 \n",
      " 10  half-true    1284 non-null   int64 \n",
      " 11  mostly-true  1284 non-null   int64 \n",
      " 12  pants-fire   1284 non-null   int64 \n",
      " 13  venue        1272 non-null   object\n",
      "dtypes: int64(5), object(9)\n",
      "memory usage: 140.6+ KB\n",
      "None\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1267 entries, 0 to 1266\n",
      "Data columns (total 14 columns):\n",
      " #   Column       Non-Null Count  Dtype \n",
      "---  ------       --------------  ----- \n",
      " 0   id           1267 non-null   object\n",
      " 1   label        1267 non-null   object\n",
      " 2   statement    1267 non-null   object\n",
      " 3   subject      1267 non-null   object\n",
      " 4   speaker      1267 non-null   object\n",
      " 5   job          942 non-null    object\n",
      " 6   state        1005 non-null   object\n",
      " 7   party        1267 non-null   object\n",
      " 8   barely-true  1267 non-null   int64 \n",
      " 9   false        1267 non-null   int64 \n",
      " 10  half-true    1267 non-null   int64 \n",
      " 11  mostly-true  1267 non-null   int64 \n",
      " 12  pants-fire   1267 non-null   int64 \n",
      " 13  venue        1250 non-null   object\n",
      "dtypes: int64(5), object(9)\n",
      "memory usage: 138.7+ KB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(train_data.info())\n",
    "print (val_data.info())\n",
    "print (test_data.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'pants-fire': 0, 'false': 1, 'barely-true': 2, 'half-true': 3, 'mostly-true': 4, 'true': 5}\n"
     ]
    }
   ],
   "source": [
    "y_label_dict = {\"pants-fire\" : 0, \"false\" : 1, \"barely-true\" : 2, \"half-true\" : 3, \"mostly-true\" : 4, \"true\" : 5}\n",
    "print (y_label_dict)\n",
    "\n",
    "train_data['output'] = train_data['label'].apply(lambda x: y_label_dict[x])\n",
    "val_data['output'] = val_data['label'].apply(lambda x: y_label_dict[x])\n",
    "test_data['output'] = test_data['label'].apply(lambda x: y_label_dict[x])\n",
    "\n",
    "num_classes = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'barack-obama': 0, 'donald-trump': 1, 'hillary-clinton': 2, 'mitt-romney': 3, 'scott-walker': 4, 'john-mccain': 5, 'rick-perry': 6, 'chain-email': 7, 'marco-rubio': 8, 'viral-image': 13, 'rick-scott': 9, 'ted-cruz': 10, 'bernie-s': 11, 'newt-gingrich': 16, 'chris-christie': 12, 'facebook-posts': 13, 'blog-posting': 13, 'charlie-crist': 14, 'congressional': 15, 'republican': 15, 'national-committe': 15, 'democratic': 15}\n",
      "17\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "17    7348\n",
       "0     488 \n",
       "15    347 \n",
       "1     275 \n",
       "2     239 \n",
       "3     176 \n",
       "13    156 \n",
       "4     149 \n",
       "5     148 \n",
       "6     142 \n",
       "7     142 \n",
       "8     117 \n",
       "9     115 \n",
       "10    93  \n",
       "11    88  \n",
       "12    78  \n",
       "14    70  \n",
       "16    69  \n",
       "Name: speaker_id, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frequent_speakers = {'barack-obama' : 0, 'donald-trump' : 1, 'hillary-clinton' : 2, \n",
    "                     'mitt-romney' : 3, 'scott-walker' : 4, 'john-mccain' : 5, \n",
    "                     'rick-perry' : 6, 'chain-email' : 7, 'marco-rubio' : 8, 'viral-image':13,\n",
    "                     'rick-scott' : 9, 'ted-cruz' : 10, 'bernie-s' : 11, 'newt-gingrich':16,\n",
    "                     'chris-christie' : 12, 'facebook-posts' : 13,'blog-posting':13, \n",
    "                     'charlie-crist' : 14, 'congressional' : 15, 'republican' : 15, \n",
    "                     'national-committe' : 15, 'democratic':15}\n",
    "\n",
    "print (frequent_speakers)\n",
    "\n",
    "def get_speaker_id(speaker):\n",
    "  if isinstance(speaker, str):\n",
    "    matched = [sp for sp in frequent_speakers if sp in speaker.lower() ]\n",
    "    if len(matched)>0:\n",
    "      return frequent_speakers[matched[0]]\n",
    "    else:\n",
    "      return len(set(frequent_speakers.values())) \n",
    "  else:\n",
    "    return len(set(frequent_speakers.values())) \n",
    "  \n",
    "\n",
    "train_data['speaker_id'] = train_data['speaker'].apply(get_speaker_id)\n",
    "val_data['speaker_id'] = val_data['speaker'].apply(get_speaker_id)\n",
    "test_data['speaker_id'] = test_data['speaker'].apply(get_speaker_id)\n",
    "\n",
    "print (len(set(frequent_speakers.values()))) \n",
    "\n",
    "train_data['speaker_id'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'senator': 0, 'president': 1, 'governor': 2, 'u.s. representative': 3, 'attorney': 4, 'congressman': 5, 'congresswoman': 5, 'social media posting': 6, 'lawyer': 4, 'businessman': 6, 'radio host': 8, 'host': 8, 'mayor': 7, 'assembly': 9, 'representative': 3, 'senate': 9, 'state representative': 10, 'milwaukee county executive': 11, 'u.s. house of representatives': 3, 'house representative': 3, 'house of representatives': 3, 'house member': 3}\n",
      "12\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "12    4597\n",
       "1     1212\n",
       "0     1201\n",
       "3     911 \n",
       "2     892 \n",
       "8     279 \n",
       "9     253 \n",
       "5     232 \n",
       "4     223 \n",
       "7     167 \n",
       "11    149 \n",
       "6     124 \n",
       "Name: job_id, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frequent_jobs = { 'senator' : 0, 'president' : 1, 'governor' : 2, \n",
    "                 'u.s. representative' : 3, 'attorney' : 4, 'congressman' : 5, \n",
    "                 'congresswoman' : 5, 'social media posting' : 6, 'lawyer' : 4, \n",
    "                 'businessman' : 6,  'radio host' : 8, 'host':8,\n",
    "                  'mayor' : 7, 'assembly' : 9,'representative' : 3, \n",
    "                 'senate' : 9,'state representative' : 10,'milwaukee county executive' : 11,\n",
    "                 'u.s. house of representatives' : 3,'house representative' : 3,\n",
    "                 'house of representatives' : 3,'house member':3}\n",
    "\n",
    "\n",
    "print (frequent_jobs)\n",
    "\n",
    "def get_job_id(job):\n",
    "  if isinstance(job, str):\n",
    "    matched = [jb for jb in frequent_jobs if jb in job.lower() ]\n",
    "    if len(matched)>0:\n",
    "      return frequent_jobs[matched[0]]\n",
    "    else:\n",
    "      return len(set(frequent_jobs.values()))\n",
    "  else:\n",
    "    return len(set(frequent_jobs.values()))\n",
    "  \n",
    "\n",
    "train_data['job_id'] = train_data['job'].apply(get_job_id)\n",
    "val_data['job_id'] = val_data['job'].apply(get_job_id)\n",
    "test_data['job_id'] = test_data['job'].apply(get_job_id)\n",
    "\n",
    "print (len(set(frequent_jobs.values())))\n",
    "\n",
    "train_data['job_id'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'republican': 0, 'democrat': 1, 'none': 2, 'organization': 3, 'independent': 4}\n",
      "5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    4497\n",
       "1    3337\n",
       "2    1744\n",
       "5    296 \n",
       "3    219 \n",
       "4    147 \n",
       "Name: party_id, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frequent_parties = train_data['party'].str.lower().value_counts()[:5].reset_index().to_dict()['index']\n",
    "frequent_parties = dict((v,k) for k,v in frequent_parties.items())\n",
    "print (frequent_parties)\n",
    "#frequent_parties['columnist']=frequent_parties['journalist']\n",
    "#frequent_parties['talk-show-host']=frequent_parties['journalist']\n",
    "def get_party_id(party):\n",
    "  if isinstance(party, str):\n",
    "    matched = [pt for pt in frequent_parties if pt in party.lower() ]\n",
    "    if len(matched)>0:\n",
    "      return frequent_parties[matched[0]]\n",
    "    else:\n",
    "      return len(set(frequent_parties.values())) \n",
    "  else:\n",
    "    return len(set(frequent_parties.values())) \n",
    "  \n",
    "\n",
    "train_data['party_id'] = train_data['party'].apply(get_party_id)\n",
    "val_data['party_id'] = val_data['party'].apply(get_party_id)\n",
    "test_data['party_id'] = test_data['party'].apply(get_party_id)\n",
    "\n",
    "print (len(set(frequent_parties.values())) )\n",
    "\n",
    "train_data['party_id'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: party, dtype: int64)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.loc[train_data['party_id']==9]['party'].value_counts()[:90]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'texas': 1, 'florida': 2, 'wisconsin': 3, 'new york': 4, 'illinois': 5, 'ohio': 6, 'georgia': 7, 'virginia': 8, 'rhode island': 9, 'oregon': 10, 'new jersey': 11, 'massachusetts': 12, 'arizona': 13, 'california': 14, 'washington': 15, 'wyoming': 0, 'colorado': 0, 'hawaii': 0, 'tennessee': 0, 'nevada': 0, 'maine': 0, 'north dakota': 0, 'mississippi': 0, 'south dakota': 0, 'oklahoma': 0, 'delaware': 0, 'minnesota': 0, 'north carolina': 0, 'arkansas': 0, 'indiana': 0, 'maryland': 0, 'louisiana': 0, 'idaho': 0, 'iowa': 0, 'west virginia': 0, 'michigan': 0, 'kansas': 0, 'utah': 0, 'connecticut': 0, 'montana': 0, 'vermont': 0, 'pennsylvania': 0, 'alaska': 0, 'kentucky': 0, 'nebraska': 0, 'new hampshire': 0, 'missouri': 0, 'south carolina': 0, 'alabama': 0, 'new mexico': 0}\n",
      "16\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "16    2238\n",
       "0     1211\n",
       "1     1009\n",
       "2     1003\n",
       "3     714 \n",
       "4     659 \n",
       "5     558 \n",
       "6     448 \n",
       "7     433 \n",
       "8     408 \n",
       "9     371 \n",
       "10    242 \n",
       "11    241 \n",
       "12    212 \n",
       "13    182 \n",
       "14    163 \n",
       "15    148 \n",
       "Name: state_id, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other_states = ['wyoming', 'colorado', 'hawaii', 'tennessee', 'nevada', 'maine',\n",
    "                'north dakota', 'mississippi', 'south dakota', 'oklahoma', \n",
    "                'delaware', 'minnesota', 'north carolina', 'arkansas', 'indiana', \n",
    "                'maryland', 'louisiana', 'idaho', 'iowa', 'west virginia', \n",
    "                'michigan', 'kansas', 'utah', 'connecticut', 'montana', 'vermont', \n",
    "                'pennsylvania', 'alaska', 'kentucky', 'nebraska', 'new hampshire', \n",
    "                'missouri', 'south carolina', 'alabama', 'new mexico']\n",
    "\n",
    "\n",
    "frequent_states = {'texas': 1, 'florida': 2, 'wisconsin': 3, 'new york': 4, \n",
    "                    'illinois': 5, 'ohio': 6, 'georgia': 7, 'virginia': 8, \n",
    "                   'rhode island': 9, 'oregon': 10, 'new jersey': 11, \n",
    "                   'massachusetts': 12, 'arizona': 13, 'california': 14, \n",
    "                   'washington': 15}\n",
    "for state in other_states:\n",
    "  frequent_states[state]=0\n",
    "\n",
    "print (frequent_states)\n",
    "\n",
    "def get_state_id(state):\n",
    "    if isinstance(state,str):\n",
    "        if state.lower().rstrip() in frequent_states:\n",
    "            return frequent_states[state.lower().rstrip()]\n",
    "        else:\n",
    "            if 'washington' in state.lower():\n",
    "                return frequent_states['washington']\n",
    "            else:\n",
    "                return len(set(frequent_states.values()))\n",
    "    else:\n",
    "        return len(set(frequent_states.values()))\n",
    "\n",
    "train_data['state_id'] = train_data['state'].apply(get_state_id)\n",
    "val_data['state_id'] = val_data['state'].apply(get_state_id)\n",
    "test_data['state_id'] = test_data['state'].apply(get_state_id)\n",
    "\n",
    "print (len(set(frequent_states.values())))\n",
    "\n",
    "train_data['state_id'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'health': 0, 'tax': 1, 'immigration': 2, 'election': 3, 'education': 4, 'candidates-biography': 5, 'economy': 6, 'gun': 7, 'job': 8, 'federal-budget': 6, 'energy': 9, 'abortion': 10, 'foreign-policy': 6, 'state-budget': 6, 'crime': 11, 'gays-and-lesbians': 12, 'medicare': 0, 'terrorism': 11, 'finance': 6, 'criminal': 11, 'transportation': 13}\n",
      "14\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "6     2103\n",
       "14    1909\n",
       "0     1302\n",
       "1     906 \n",
       "4     621 \n",
       "3     569 \n",
       "5     512 \n",
       "2     506 \n",
       "11    438 \n",
       "8     409 \n",
       "9     305 \n",
       "7     278 \n",
       "10    170 \n",
       "13    127 \n",
       "12    85  \n",
       "Name: subject_id, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frequent_subjects = {'health': 0, 'tax': 1, 'immigration': 2, 'election': 3, \n",
    "                     'education': 4, 'candidates-biography': 5, 'economy': 6, \n",
    "                     'gun': 7, 'job': 8, 'federal-budget': 6, 'energy': 9, \n",
    "                     'abortion': 10, 'foreign-policy': 6, 'state-budget': 6, \n",
    "                     'crime': 11, 'gays-and-lesbians' : 12, 'medicare' : 0, \n",
    "                     'terrorism' : 11, 'finance' : 6, 'criminal':11,\n",
    "                     'transportation':13}\n",
    "\n",
    "print (frequent_subjects)\n",
    "\n",
    "\n",
    "def get_subject_id(subject):\n",
    "  if isinstance(subject, str):\n",
    "    matched = [sbj for sbj in frequent_subjects if sbj in subject.lower() ]\n",
    "    if len(matched)>0:\n",
    "      return frequent_subjects[matched[0]]\n",
    "    else:\n",
    "      return len(set(frequent_subjects.values())) \n",
    "  else:\n",
    "    return len(set(frequent_subjects.values()))\n",
    "  \n",
    "\n",
    "train_data['subject_id'] = train_data['subject'].apply(get_subject_id)\n",
    "val_data['subject_id'] = val_data['subject'].apply(get_subject_id)\n",
    "test_data['subject_id'] = test_data['subject'].apply(get_subject_id)\n",
    "\n",
    "print (len(set(frequent_subjects.values())))\n",
    "\n",
    "train_data['subject_id'].value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'news release': 0, 'interview': 1, 'press release': 2, 'speech': 3, 'tv': 4, 'tweet': 5, 'campaign': 6, 'television': 4, 'debate': 7, 'news conference': 8, 'facebook': 5, 'press conference': 8, 'radio': 9, 'e-mail': 10, 'email': 10, 'mail': 10, 'social media': 5, 'twitter': 5, 'blog': 11, 'article': 11, 'comment': 12, 'web': 11}\n",
      "13\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "13    2695\n",
       "1     1752\n",
       "3     1059\n",
       "7     735 \n",
       "6     679 \n",
       "4     568 \n",
       "11    529 \n",
       "5     473 \n",
       "10    356 \n",
       "2     346 \n",
       "12    337 \n",
       "0     324 \n",
       "8     249 \n",
       "9     138 \n",
       "Name: venue_id, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frequent_venues = {'news release' : 0, 'interview' : 1, 'press release' : 2, \n",
    "                   'speech' : 3, 'tv' : 4, 'tweet' : 5, 'campaign' : 6, \n",
    "                   'television' : 4, 'debate' : 7, 'news conference' : 8, \n",
    "                   'facebook' : 5, 'press conference' : 8, 'radio' : 9, \n",
    "                   'e-mail' : 10, 'email' : 10, 'mail' : 10, 'social media' : 5,\n",
    "                   'twitter' : 5, 'blog':11, 'article':11,'comment':12, 'web':11}\n",
    "\n",
    "print (frequent_venues)\n",
    "\n",
    "\n",
    "def get_venue_id(venue):\n",
    "  if isinstance(venue, str):\n",
    "    matched = [ven for ven in frequent_venues if ven in venue.lower() ]\n",
    "    if len(matched)>0:\n",
    "      return frequent_venues[matched[0]]\n",
    "    else:\n",
    "      return len(set(frequent_venues.values())) \n",
    "  else:\n",
    "    return len(set(frequent_venues.values()))\n",
    "  \n",
    "\n",
    "train_data['venue_id'] = train_data['venue'].apply(get_venue_id)\n",
    "val_data['venue_id'] = val_data['venue'].apply(get_venue_id)\n",
    "test_data['venue_id'] = test_data['venue'].apply(get_venue_id)\n",
    "\n",
    "print (len(set(frequent_venues.values())))\n",
    "\n",
    "train_data['venue_id'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>statement</th>\n",
       "      <th>subject</th>\n",
       "      <th>speaker</th>\n",
       "      <th>job</th>\n",
       "      <th>state</th>\n",
       "      <th>party</th>\n",
       "      <th>barely-true</th>\n",
       "      <th>false</th>\n",
       "      <th>...</th>\n",
       "      <th>mostly-true</th>\n",
       "      <th>pants-fire</th>\n",
       "      <th>venue</th>\n",
       "      <th>output</th>\n",
       "      <th>speaker_id</th>\n",
       "      <th>job_id</th>\n",
       "      <th>party_id</th>\n",
       "      <th>state_id</th>\n",
       "      <th>subject_id</th>\n",
       "      <th>venue_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2635.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Says the Annies List political group supports third-trimester abortions on demand.</td>\n",
       "      <td>abortion</td>\n",
       "      <td>dwayne-bohac</td>\n",
       "      <td>State representative</td>\n",
       "      <td>Texas</td>\n",
       "      <td>republican</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>a mailer</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10540.json</td>\n",
       "      <td>half-true</td>\n",
       "      <td>When did the decline of coal start? It started when natural gas took off that started to begin in (President George W.) Bushs administration.</td>\n",
       "      <td>energy,history,job-accomplishments</td>\n",
       "      <td>scott-surovell</td>\n",
       "      <td>State delegate</td>\n",
       "      <td>Virginia</td>\n",
       "      <td>democrat</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>a floor speech.</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>324.json</td>\n",
       "      <td>mostly-true</td>\n",
       "      <td>Hillary Clinton agrees with John McCain \"by voting to give George Bush the benefit of the doubt on Iran.\"</td>\n",
       "      <td>foreign-policy</td>\n",
       "      <td>barack-obama</td>\n",
       "      <td>President</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>democrat</td>\n",
       "      <td>70.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>...</td>\n",
       "      <td>163.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Denver</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1123.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Health care reform legislation is likely to mandate free sex change surgeries.</td>\n",
       "      <td>health-care</td>\n",
       "      <td>blog-posting</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>none</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>a news release</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9028.json</td>\n",
       "      <td>half-true</td>\n",
       "      <td>The economic turnaround started at the end of my term.</td>\n",
       "      <td>economy,jobs</td>\n",
       "      <td>charlie-crist</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Florida</td>\n",
       "      <td>democrat</td>\n",
       "      <td>15.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>...</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>an interview on CNN</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id        label  \\\n",
       "0  2635.json   false         \n",
       "1  10540.json  half-true     \n",
       "2  324.json    mostly-true   \n",
       "3  1123.json   false         \n",
       "4  9028.json   half-true     \n",
       "\n",
       "                                                                                                                                       statement  \\\n",
       "0  Says the Annies List political group supports third-trimester abortions on demand.                                                              \n",
       "1  When did the decline of coal start? It started when natural gas took off that started to begin in (President George W.) Bushs administration.   \n",
       "2  Hillary Clinton agrees with John McCain \"by voting to give George Bush the benefit of the doubt on Iran.\"                                       \n",
       "3  Health care reform legislation is likely to mandate free sex change surgeries.                                                                  \n",
       "4  The economic turnaround started at the end of my term.                                                                                          \n",
       "\n",
       "                              subject         speaker                   job  \\\n",
       "0  abortion                            dwayne-bohac    State representative   \n",
       "1  energy,history,job-accomplishments  scott-surovell  State delegate         \n",
       "2  foreign-policy                      barack-obama    President              \n",
       "3  health-care                         blog-posting    NaN                    \n",
       "4  economy,jobs                        charlie-crist   NaN                    \n",
       "\n",
       "      state       party  barely-true  false  ...  mostly-true  pants-fire  \\\n",
       "0  Texas     republican  0.0          1.0    ...  0.0          0.0          \n",
       "1  Virginia  democrat    0.0          0.0    ...  1.0          0.0          \n",
       "2  Illinois  democrat    70.0         71.0   ...  163.0        9.0          \n",
       "3  NaN       none        7.0          19.0   ...  5.0          44.0         \n",
       "4  Florida   democrat    15.0         9.0    ...  19.0         2.0          \n",
       "\n",
       "                 venue output  speaker_id  job_id  party_id  state_id  \\\n",
       "0  a mailer             1      17          3       0         1          \n",
       "1  a floor speech.      3      17          12      1         8          \n",
       "2  Denver               4      0           1       1         5          \n",
       "3  a news release       1      13          12      2         16         \n",
       "4  an interview on CNN  3      14          12      1         2          \n",
       "\n",
       "   subject_id  venue_id  \n",
       "0  10          10        \n",
       "1  8           3         \n",
       "2  6           13        \n",
       "3  0           0         \n",
       "4  6           1         \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Vocabulary Dictionary...\n"
     ]
    }
   ],
   "source": [
    "def load_statement_vocab_dict(train_data):\n",
    "  vocabulary_dict = {}\n",
    "  if not os.path.exists('vocabulary.p'):\n",
    "    tokenizer = Tokenizer()\n",
    "    tokenizer.fit_on_texts(train_data['statement'])\n",
    "    vocabulary_dict = tokenizer.word_index\n",
    "    print (len(vocabulary_dict))\n",
    "    pickle.dump(vocabulary_dict, open( \"vocabulary.p\", \"wb\" ))\n",
    "    print ('Created Vocabulary Dictionary...')\n",
    "    print ('Saved Vocabulary Dictionary...')\n",
    "  else:\n",
    "    print ('Loading Vocabulary Dictionary...')\n",
    "    vocabulary_dict = pickle.load(open(\"vocabulary.p\", \"rb\" ))\n",
    "  return vocabulary_dict\n",
    "\n",
    "\n",
    "def preprocess_statement(statement):\n",
    "  statement = [w for w in statement.split(' ') if w not in stopwords.words('english')]\n",
    "  statement = ' '.join(statement)\n",
    "  text = text_to_word_sequence(statement)\n",
    "  val = [0] * 10\n",
    "  val = [vocabulary_dict[t] for t in text if t in vocabulary_dict] \n",
    "  return val\n",
    "\n",
    "vocabulary_dict = load_statement_vocab_dict(train_data)\n",
    "train_data['word_id'] = train_data['statement'].apply(preprocess_statement)\n",
    "val_data['word_id'] = val_data['statement'].apply(preprocess_statement)\n",
    "test_data['word_id'] = test_data['statement'].apply(preprocess_statement)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_tags = {'ADJ': 'adjective', 'ADP': 'adposition', 'ADV': 'adverb', \n",
    "            'AUX': 'auxiliary verb', 'CONJ': 'coordinating conjunction', \n",
    "            'DET': 'determiner', 'INTJ': 'interjection', 'NOUN': 'noun', \n",
    "            'NUM': 'numeral', 'PART': 'particle', 'PRON': 'pronoun', \n",
    "            'PROPN': 'proper noun', 'PUNCT': 'punctuation', 'X': 'other', \n",
    "            'SCONJ': 'subord conjunction', 'SYM': 'symbol', 'VERB': 'verb'}\n",
    "\n",
    "#pos_dict = {'ADJ' : 0, 'ADP' : 1, 'ADV' : 2, 'AUX' : 3, 'CONJ' : 4, 'DET' : 5, \n",
    "#            'INTJ' : 6, 'NOUN' : 7, 'NUM' : 8, 'PART' : 9, 'PRON' : 10, 'X' : 16,\n",
    "#            'PROPN' : 11, 'PUNCT' : 12, 'SCONJ' : 13, 'SYM' : 14, 'VERB' : 15}\n",
    "\n",
    "pos_dict = {'NOUN' : 0, 'VERB' : 1, 'ADP' : 2, 'PROPN' : 3, 'PUNCT' : 4, \n",
    "            'DET' : 5, 'ADJ' : 6, 'NUM' : 7, 'ADV' : 8, 'PRON' : 9, 'X' : 9, \n",
    "            'PART' : 9, 'SYM' : 9, 'INTJ' : 9 }\n",
    "\n",
    "dep_dict = {'ACL' : 0, 'ACOMP' : 1, 'ADVCL' : 2, 'ADVMOD' : 3, 'AGENT' : 4, \n",
    "            'AMOD' : 5, 'APPOS' : 6, 'ATTR' : 7, 'AUX' : 8, 'AUXPASS' : 9, \n",
    "            'CASE' : 10, 'CC' : 11, 'CCOMP' : 12, 'COMPOUND' : 13, 'CONJ' : 14, \n",
    "            'CSUBJ' : 15, 'CSUBJPASS' : 16, 'DATIVE' : 17, 'DEP' : 18, \n",
    "            'DET' : 19, 'DOBJ' : 20, 'EXPL' : 21, 'INTJ' : 22, 'MARK' : 23, \n",
    "            'META' : 24, 'NEG' : 25, 'NOUNMOD' : 26, 'NPMOD' : 27, 'NSUBJ' : 28, \n",
    "            'NSUBJPASS' : 29, 'NUMMOD' : 30, 'OPRD' : 31, 'PARATAXIS' : 32, \n",
    "            'PCOMP' : 33, 'POBJ' : 34, 'POSS' : 35, 'PRECONJ' : 36, 'PREDET' : 37, \n",
    "            'PREP' : 38, 'PRT' : 39, 'PUNCT' : 40, 'QUANTMOD' : 41, \n",
    "            'RELCL' : 42, 'ROOT' : 43, 'XCOMP' : 44}\n",
    "\n",
    "def get_pos(statement):\n",
    "  doc = nlp(statement)\n",
    "  taglist = []\n",
    "  deplist = []\n",
    "  for token in doc:\n",
    "    taglist.append(pos_dict.get(token.pos_,max(pos_dict.values())))\n",
    "    #deplist.append(token.dep_)\n",
    "  return taglist\n",
    "train_data['pos_id'] = train_data['statement'].apply(get_pos)\n",
    "val_data['pos_id'] = val_data['statement'].apply(get_pos)\n",
    "test_data['pos_id'] = test_data['statement'].apply(get_pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>statement</th>\n",
       "      <th>subject</th>\n",
       "      <th>speaker</th>\n",
       "      <th>job</th>\n",
       "      <th>state</th>\n",
       "      <th>party</th>\n",
       "      <th>barely-true</th>\n",
       "      <th>false</th>\n",
       "      <th>...</th>\n",
       "      <th>output</th>\n",
       "      <th>speaker_id</th>\n",
       "      <th>job_id</th>\n",
       "      <th>party_id</th>\n",
       "      <th>state_id</th>\n",
       "      <th>subject_id</th>\n",
       "      <th>venue_id</th>\n",
       "      <th>word_id</th>\n",
       "      <th>pos_id</th>\n",
       "      <th>dep_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2635.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Says the Annies List political group supports third-trimester abortions on demand.</td>\n",
       "      <td>abortion</td>\n",
       "      <td>dwayne-bohac</td>\n",
       "      <td>State representative</td>\n",
       "      <td>Texas</td>\n",
       "      <td>republican</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>[7, 6968, 1141, 520, 621, 385, 444, 5119, 585, 1601]</td>\n",
       "      <td>[1, 5, 3, 3, 6, 0, 1, 6, 4, 0, 0, 2, 0, 4]</td>\n",
       "      <td>[6, 4, 3, 10, 7, 5, 10, 7, 0, 3, 8, 1, 2, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10540.json</td>\n",
       "      <td>half-true</td>\n",
       "      <td>When did the decline of coal start? It started when natural gas took off that started to begin in (President George W.) Bushs administration.</td>\n",
       "      <td>energy,history,job-accomplishments</td>\n",
       "      <td>scott-surovell</td>\n",
       "      <td>State delegate</td>\n",
       "      <td>Virginia</td>\n",
       "      <td>democrat</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>[63, 2091, 964, 866, 23, 602, 1142, 315, 180, 602, 1959, 34, 310, 560, 1365, 177]</td>\n",
       "      <td>[8, 9, 5, 0, 2, 0, 0, 4, 9, 1, 8, 6, 0, 1, 2, 5, 1, 9, 1, 2, 4, 3, 3, 3, 4, 3, 0, 4]</td>\n",
       "      <td>[10, 6, 4, 5, 1, 3, 2, 0, 5, 6, 10, 7, 5, 10, 10, 5, 10, 9, 10, 1, 0, 3, 3, 2, 0, 3, 6, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>324.json</td>\n",
       "      <td>mostly-true</td>\n",
       "      <td>Hillary Clinton agrees with John McCain \"by voting to give George Bush the benefit of the doubt on Iran.\"</td>\n",
       "      <td>foreign-policy</td>\n",
       "      <td>barack-obama</td>\n",
       "      <td>President</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>democrat</td>\n",
       "      <td>70.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>[127, 101, 3546, 191, 254, 20, 329, 343, 310, 166, 1093, 3547, 416]</td>\n",
       "      <td>[3, 3, 1, 2, 3, 3, 4, 2, 1, 9, 1, 3, 3, 5, 0, 2, 5, 0, 2, 3, 4, 4]</td>\n",
       "      <td>[3, 5, 6, 1, 3, 2, 0, 1, 10, 9, 10, 3, 10, 4, 8, 1, 4, 2, 1, 2, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1123.json</td>\n",
       "      <td>false</td>\n",
       "      <td>Health care reform legislation is likely to mandate free sex change surgeries.</td>\n",
       "      <td>health-care</td>\n",
       "      <td>blog-posting</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>none</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[32, 43, 266, 298, 666, 667, 404, 467, 417, 4148]</td>\n",
       "      <td>[0, 0, 0, 0, 9, 6, 9, 1, 6, 0, 0, 0, 4]</td>\n",
       "      <td>[3, 3, 3, 5, 6, 10, 9, 10, 7, 3, 3, 8, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9028.json</td>\n",
       "      <td>half-true</td>\n",
       "      <td>The economic turnaround started at the end of my term.</td>\n",
       "      <td>economy,jobs</td>\n",
       "      <td>charlie-crist</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Florida</td>\n",
       "      <td>democrat</td>\n",
       "      <td>15.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>[1, 325, 4149, 602, 408, 505]</td>\n",
       "      <td>[5, 6, 0, 1, 2, 5, 0, 2, 5, 0, 4]</td>\n",
       "      <td>[4, 7, 5, 6, 1, 4, 2, 1, 10, 2, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id        label  \\\n",
       "0  2635.json   false         \n",
       "1  10540.json  half-true     \n",
       "2  324.json    mostly-true   \n",
       "3  1123.json   false         \n",
       "4  9028.json   half-true     \n",
       "\n",
       "                                                                                                                                       statement  \\\n",
       "0  Says the Annies List political group supports third-trimester abortions on demand.                                                              \n",
       "1  When did the decline of coal start? It started when natural gas took off that started to begin in (President George W.) Bushs administration.   \n",
       "2  Hillary Clinton agrees with John McCain \"by voting to give George Bush the benefit of the doubt on Iran.\"                                       \n",
       "3  Health care reform legislation is likely to mandate free sex change surgeries.                                                                  \n",
       "4  The economic turnaround started at the end of my term.                                                                                          \n",
       "\n",
       "                              subject         speaker                   job  \\\n",
       "0  abortion                            dwayne-bohac    State representative   \n",
       "1  energy,history,job-accomplishments  scott-surovell  State delegate         \n",
       "2  foreign-policy                      barack-obama    President              \n",
       "3  health-care                         blog-posting    NaN                    \n",
       "4  economy,jobs                        charlie-crist   NaN                    \n",
       "\n",
       "      state       party  barely-true  false  ...  output  speaker_id  job_id  \\\n",
       "0  Texas     republican  0.0          1.0    ...  1       17          3        \n",
       "1  Virginia  democrat    0.0          0.0    ...  3       17          12       \n",
       "2  Illinois  democrat    70.0         71.0   ...  4       0           1        \n",
       "3  NaN       none        7.0          19.0   ...  1       13          12       \n",
       "4  Florida   democrat    15.0         9.0    ...  3       14          12       \n",
       "\n",
       "  party_id  state_id  subject_id  venue_id  \\\n",
       "0  0        1         10          10         \n",
       "1  1        8         8           3          \n",
       "2  1        5         6           13         \n",
       "3  2        16        0           0          \n",
       "4  1        2         6           1          \n",
       "\n",
       "                                                                             word_id  \\\n",
       "0  [7, 6968, 1141, 520, 621, 385, 444, 5119, 585, 1601]                                \n",
       "1  [63, 2091, 964, 866, 23, 602, 1142, 315, 180, 602, 1959, 34, 310, 560, 1365, 177]   \n",
       "2  [127, 101, 3546, 191, 254, 20, 329, 343, 310, 166, 1093, 3547, 416]                 \n",
       "3  [32, 43, 266, 298, 666, 667, 404, 467, 417, 4148]                                   \n",
       "4  [1, 325, 4149, 602, 408, 505]                                                       \n",
       "\n",
       "                                                                                 pos_id  \\\n",
       "0  [1, 5, 3, 3, 6, 0, 1, 6, 4, 0, 0, 2, 0, 4]                                             \n",
       "1  [8, 9, 5, 0, 2, 0, 0, 4, 9, 1, 8, 6, 0, 1, 2, 5, 1, 9, 1, 2, 4, 3, 3, 3, 4, 3, 0, 4]   \n",
       "2  [3, 3, 1, 2, 3, 3, 4, 2, 1, 9, 1, 3, 3, 5, 0, 2, 5, 0, 2, 3, 4, 4]                     \n",
       "3  [0, 0, 0, 0, 9, 6, 9, 1, 6, 0, 0, 0, 4]                                                \n",
       "4  [5, 6, 0, 1, 2, 5, 0, 2, 5, 0, 4]                                                      \n",
       "\n",
       "                                                                                       dep_id  \n",
       "0  [6, 4, 3, 10, 7, 5, 10, 7, 0, 3, 8, 1, 2, 0]                                                \n",
       "1  [10, 6, 4, 5, 1, 3, 2, 0, 5, 6, 10, 7, 5, 10, 10, 5, 10, 9, 10, 1, 0, 3, 3, 2, 0, 3, 6, 0]  \n",
       "2  [3, 5, 6, 1, 3, 2, 0, 1, 10, 9, 10, 3, 10, 4, 8, 1, 4, 2, 1, 2, 0, 0]                       \n",
       "3  [3, 3, 3, 5, 6, 10, 9, 10, 7, 3, 3, 8, 0]                                                   \n",
       "4  [4, 7, 5, 6, 1, 4, 2, 1, 10, 2, 0]                                                          \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dep_dict = {'ACL' : 0, 'ACOMP' : 1, 'ADVCL' : 2, 'ADVMOD' : 3, 'AGENT' : 4, \n",
    "            'AMOD' : 5, 'APPOS' : 6, 'ATTR' : 7, 'AUX' : 8, 'AUXPASS' : 9, \n",
    "            'CASE' : 10, 'CC' : 11, 'CCOMP' : 12, 'COMPOUND' : 13, 'CONJ' : 14, \n",
    "            'CSUBJ' : 15, 'CSUBJPASS' : 16, 'DATIVE' : 17, 'DEP' : 18, \n",
    "            'DET' : 19, 'DOBJ' : 20, 'EXPL' : 21, 'INTJ' : 22, 'MARK' : 23, \n",
    "            'META' : 24, 'NEG' : 25, 'NOUNMOD' : 26, 'NPMOD' : 27, 'NSUBJ' : 28, \n",
    "            'NSUBJPASS' : 29, 'NUMMOD' : 30, 'OPRD' : 31, 'PARATAXIS' : 32, \n",
    "            'PCOMP' : 33, 'POBJ' : 34, 'POSS' : 35, 'PRECONJ' : 36, 'PREDET' : 37, \n",
    "            'PREP' : 38, 'PRT' : 39, 'PUNCT' : 40, 'QUANTMOD' : 41, \n",
    "            'RELCL' : 42, 'ROOT' : 43, 'XCOMP' : 44}\n",
    "\n",
    "\n",
    "dep_dict = {'punct' : 0, 'prep' : 1, 'pobj' : 2, 'compound' : 3, 'det' : 4, \n",
    "            'nsubj' : 5, 'ROOT' : 6, 'amod' : 7, 'dobj' : 8, 'aux' : 9, \n",
    "            'advmod' : 10, 'nummod' : 10, 'ccomp' : 10, 'conj' : 10, 'cc' : 10, \n",
    "            'advcl' : 10, 'poss' : 10, 'mark' : 10, 'quantmod' : 10, 'relcl' : 10, \n",
    "            'attr' : 10, 'xcomp' : 10, 'npadvmod' : 10, 'nmod' : 10, 'auxpass' : 10, \n",
    "            'acl' : 10, 'nsubjpass' : 10, 'pcomp' : 10, 'acomp' : 10, 'neg' : 10, \n",
    "            'appos' : 10, 'prt' : 10, '' : 10, 'expl' : 10, 'dative' : 10, \n",
    "            'agent' : 10, 'case' : 10, 'oprd' : 10, 'csubj' : 10, 'dep' : 10, \n",
    "            'intj' : 10, 'predet' : 10, 'parataxis' : 10, 'preconj' : 10, \n",
    "            'meta' : 10, 'csubjpass' : 10}\n",
    "\n",
    "\n",
    "def get_dep_parse(statement):\n",
    "  doc = nlp(statement)\n",
    "  deplist = []\n",
    "  for token in doc:\n",
    "    deplist.append(dep_dict.get(token.dep_, max(dep_dict.values())))\n",
    "  return deplist\n",
    "\n",
    "\n",
    "train_data['dep_id'] = train_data['statement'].apply(get_dep_parse)\n",
    "val_data['dep_id'] = val_data['statement'].apply(get_dep_parse)\n",
    "test_data['dep_id'] = test_data['statement'].apply(get_dep_parse)\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400000  : Word Embeddings Found\n",
      "100  : Embedding Dimension\n"
     ]
    }
   ],
   "source": [
    "embeddings = {}\n",
    "with open(\"glove.6B.100d.txt\", \"br\") as file_object:\n",
    "  for line in file_object:\n",
    "    word_embed = line.split()\n",
    "    word = word_embed[0]\n",
    "    embed = np.array(word_embed[1:], dtype=\"float32\")\n",
    "    embeddings[word.lower()]= embed\n",
    "\n",
    "EMBED_DIM = 100\n",
    "print (len(embeddings), \" : Word Embeddings Found\")\n",
    "print (len(embeddings[word]), \" : Embedding Dimension\")\n",
    "\n",
    "\n",
    "num_words = len(vocabulary_dict) + 1\n",
    "embedding_matrix = np.zeros((num_words, EMBED_DIM))\n",
    "for word, i in vocabulary_dict.items():\n",
    "    embedding_vector = embeddings.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "\n",
    "embeddings_index = None\n",
    "\n",
    "\n",
    "\n",
    "pos_embeddings = np.identity(max(pos_dict.values()), dtype=int)\n",
    "dep_embeddings = np.identity(max(dep_dict.values()), dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "17\n",
      "14\n",
      "12\n",
      "15\n",
      "18\n",
      "Index(['id', 'label', 'statement', 'subject', 'speaker', 'job', 'state',\n",
      "       'party', 'barely-true', 'false', 'half-true', 'mostly-true',\n",
      "       'pants-fire', 'venue', 'output', 'speaker_id', 'job_id', 'party_id',\n",
      "       'state_id', 'subject_id', 'venue_id', 'word_id', 'pos_id', 'dep_id'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "\n",
    "vocab_length = len(vocabulary_dict.keys())\n",
    "hidden_size = EMBED_DIM #Has to be same as EMBED_DIM\n",
    "lstm_size = 100\n",
    "num_steps = 15\n",
    "num_epochs = 30\n",
    "batch_size = 40\n",
    "kernel_sizes = [3,3,3]\n",
    "filter_size = 128\n",
    "num_party = len(train_data.party_id.unique())\n",
    "num_state = len(train_data.state_id.unique())\n",
    "num_venue = len(train_data.venue_id.unique())\n",
    "num_job = len(train_data.job_id.unique())\n",
    "num_sub = len(train_data.subject_id.unique())\n",
    "num_speaker = len(train_data.speaker_id.unique())\n",
    "print (num_party)\n",
    "print (num_state)\n",
    "print (num_venue)\n",
    "print (num_job)\n",
    "print (num_sub)\n",
    "print (num_speaker)\n",
    "print (train_data.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = train_data['word_id']\n",
    "X_val = val_data['word_id']\n",
    "X_test = test_data['word_id']\n",
    "\n",
    "Y_train = train_data['output']\n",
    "Y_train = tensorflow.keras.utils.to_categorical(Y_train, num_classes=6)\n",
    "\n",
    "Y_val = val_data['output']\n",
    "Y_val = tensorflow.keras.utils.to_categorical(Y_val, num_classes=6)\n",
    "\n",
    "X_train = sequence.pad_sequences(X_train, maxlen=num_steps, padding='post',truncating='post')\n",
    "X_val = sequence.pad_sequences(X_val, maxlen=num_steps, padding='post',truncating='post')\n",
    "X_test = sequence.pad_sequences(X_test, maxlen=num_steps, padding='post',truncating='post')\n",
    "\n",
    "X_train_pos = train_data['pos_id']\n",
    "X_val_pos = val_data['pos_id']\n",
    "X_test_pos = test_data['pos_id']\n",
    "\n",
    "X_train_pos = sequence.pad_sequences(X_train_pos, maxlen=num_steps, padding='post',truncating='post')\n",
    "X_val_pos = sequence.pad_sequences(X_val_pos, maxlen=num_steps, padding='post',truncating='post')\n",
    "X_test_pos = sequence.pad_sequences(X_test_pos, maxlen=num_steps, padding='post',truncating='post')\n",
    "\n",
    "\n",
    "X_train_dep = train_data['dep_id']\n",
    "X_val_dep = val_data['dep_id']\n",
    "X_test_dep = test_data['dep_id']\n",
    "\n",
    "X_train_dep = sequence.pad_sequences(X_train_dep, maxlen=num_steps, padding='post',truncating='post')\n",
    "X_val_dep = sequence.pad_sequences(X_val_dep, maxlen=num_steps, padding='post',truncating='post')\n",
    "X_test_dep = sequence.pad_sequences(X_test_dep, maxlen=num_steps, padding='post',truncating='post')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_job=13\n",
    "#Meta data preparation\n",
    "job_train = tensorflow.keras.utils.to_categorical(train_data['job_id'], num_classes=num_job)\n",
    "party_train = tensorflow.keras.utils.to_categorical(train_data['party_id'], num_classes=num_party)\n",
    "state_train = tensorflow.keras.utils.to_categorical(train_data['state_id'], num_classes=num_state)\n",
    "venue_train = tensorflow.keras.utils.to_categorical(train_data['venue_id'], num_classes=num_venue)\n",
    "\n",
    "\n",
    "subject_train = tensorflow.keras.utils.to_categorical(train_data['subject_id'], num_classes=num_sub)\n",
    "speaker_train = tensorflow.keras.utils.to_categorical(train_data['speaker_id'], num_classes=num_speaker)\n",
    "#X_train_meta = party_train\n",
    "X_train_meta = np.hstack((party_train, state_train, venue_train, job_train, subject_train, speaker_train))\n",
    "\n",
    "party_val = tensorflow.keras.utils.to_categorical(val_data['party_id'], num_classes=num_party)\n",
    "state_val = tensorflow.keras.utils.to_categorical(val_data['state_id'], num_classes=num_state)\n",
    "venue_val = tensorflow.keras.utils.to_categorical(val_data['venue_id'], num_classes=num_venue)\n",
    "job_val = tensorflow.keras.utils.to_categorical(val_data['job_id'], num_classes=num_job)\n",
    "subject_val = tensorflow.keras.utils.to_categorical(val_data['subject_id'], num_classes=num_sub)\n",
    "speaker_val = tensorflow.keras.utils.to_categorical(val_data['speaker_id'], num_classes=num_speaker)\n",
    "\n",
    "#X_val_meta = party_val\n",
    "X_val_meta = np.hstack((party_val, state_val, venue_val, job_val, subject_val, speaker_val))\n",
    "\n",
    "party_test = tensorflow.keras.utils.to_categorical(test_data['party_id'], num_classes=num_party)\n",
    "state_test = tensorflow.keras.utils.to_categorical(test_data['state_id'], num_classes=num_state)\n",
    "venue_test = tensorflow.keras.utils.to_categorical(test_data['venue_id'], num_classes=num_venue)\n",
    "job_test = tensorflow.keras.utils.to_categorical(test_data['job_id'], num_classes=num_job)\n",
    "subject_test = tensorflow.keras.utils.to_categorical(test_data['subject_id'], num_classes=num_sub)\n",
    "speaker_test = tensorflow.keras.utils.to_categorical(test_data['speaker_id'], num_classes=num_speaker)\n",
    "\n",
    "#X_test_meta = party_test\n",
    "X_test_meta = np.hstack((party_test, state_test, venue_test, job_test, subject_test, speaker_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10240, 83) (1284, 83) (1267, 83)\n",
      "(10240, 15) (1284, 15) (1267, 15)\n",
      "(10240, 15) (1284, 15) (1267, 15)\n",
      "(10240, 15) (1284, 15) (1267, 15)\n",
      "(10240, 6) (1284, 6)\n",
      "[[ 6  4  3 ...  2  0  0]\n",
      " [10  6  4 ...  5 10 10]\n",
      " [ 3  5  6 ... 10  4  8]\n",
      " ...\n",
      " [ 6  4  5 ... 10  0  9]\n",
      " [ 6 10  4 ...  0  0  0]\n",
      " [ 4  5  1 ...  3  8  1]] [[ 5  6  7 ...  0  0  0]\n",
      " [10 10 10 ...  3  8  0]\n",
      " [ 6 10  8 ... 10  1  4]\n",
      " ...\n",
      " [ 3  5 10 ...  3  2  0]\n",
      " [ 4  7  5 ... 10  3  8]\n",
      " [ 4  5  6 ...  4  3  2]] [[10  4  8 ...  0  0  0]\n",
      " [ 5  6  1 ...  0  0  0]\n",
      " [ 6  3  5 ...  0  0  0]\n",
      " ...\n",
      " [ 1  4  7 ...  7  5 10]\n",
      " [ 6  4  3 ... 10  5 10]\n",
      " [ 6  4  5 ...  0  5  9]]\n"
     ]
    }
   ],
   "source": [
    "print (X_train_meta.shape, X_val_meta.shape, X_test_meta.shape)\n",
    "print (X_train.shape, X_val.shape, X_test.shape)\n",
    "print (X_train_pos.shape, X_val_pos.shape, X_test_pos.shape)\n",
    "print (X_train_dep.shape, X_val_dep.shape, X_test_dep.shape)\n",
    "print (Y_train.shape, Y_val.shape)\n",
    "\n",
    "print (X_train_dep, X_val_dep, X_test_dep)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "17\n",
      "14\n",
      "12\n",
      "15\n",
      "18\n",
      "Index(['id', 'label', 'statement', 'subject', 'speaker', 'job', 'state',\n",
      "       'party', 'barely-true', 'false', 'half-true', 'mostly-true',\n",
      "       'pants-fire', 'venue', 'output', 'speaker_id', 'job_id', 'party_id',\n",
      "       'state_id', 'subject_id', 'venue_id', 'word_id', 'pos_id', 'dep_id'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "vocab_length = len(vocabulary_dict.keys())\n",
    "hidden_size = EMBED_DIM #Has to be same as EMBED_DIM\n",
    "lstm_size = 100\n",
    "num_steps = 15\n",
    "num_epochs = 30\n",
    "batch_size = 40\n",
    "\n",
    "#Hyperparams for CNN\n",
    "kernel_sizes = [3,3,3]\n",
    "filter_size = 128\n",
    "\n",
    "#Meta data related hyper params\n",
    "num_party = len(train_data.party_id.unique())\n",
    "num_state = len(train_data.state_id.unique())\n",
    "num_venue = len(train_data.venue_id.unique())\n",
    "num_job = len(train_data.job_id.unique())\n",
    "num_sub = len(train_data.subject_id.unique())\n",
    "num_speaker = len(train_data.speaker_id.unique())\n",
    "print (num_party)\n",
    "print (num_state)\n",
    "print (num_venue)\n",
    "print (num_job)\n",
    "print (num_sub)\n",
    "print (num_speaker)\n",
    "print (train_data.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "128\n",
      "128\n",
      "128\n",
      "WARNING:tensorflow:Large dropout rate: 0.6 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "WARNING:tensorflow:Large dropout rate: 0.6 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "WARNING:tensorflow:Large dropout rate: 0.6 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    }
   ],
   "source": [
    "kernel_stmt = []\n",
    "kernel_pos = []\n",
    "kernel_dep = []\n",
    "\n",
    "use_pos=False\n",
    "use_meta=True\n",
    "use_dep=True\n",
    "\n",
    "statement_input = Input(shape=(num_steps,), dtype='int32', name='main_input')\n",
    "x_stmt = Embedding(vocab_length+1,EMBED_DIM,input_length=num_steps)(statement_input) \n",
    "\n",
    "# pos embed LSTM\n",
    "pos_input = Input(shape=(num_steps,), dtype='int32', name='pos_input')\n",
    "x_pos = Embedding(max(pos_dict.values()), max(pos_dict.values()), input_length=num_steps)(pos_input)\n",
    "\n",
    "# dep embed LSTM\n",
    "dep_input = Input(shape=(num_steps,), dtype='int32', name='dep_input')\n",
    "x_dep = Embedding(max(dep_dict.values()), max(dep_dict.values()), input_length=num_steps,)(dep_input)\n",
    "\n",
    "\n",
    "for kernel in kernel_sizes:\n",
    "    print(filter_size)\n",
    "    x_1 = Conv1D(filters=filter_size,kernel_size=kernel)(x_stmt)\n",
    "    x_1 = GlobalMaxPool1D()(x_1)\n",
    "    kernel_stmt.append(x_1)\n",
    "    \n",
    "    x_2 = Conv1D(filters=filter_size,kernel_size=kernel)(x_pos)\n",
    "    x_2 = GlobalMaxPool1D()(x_2)\n",
    "    kernel_pos.append(x_2)\n",
    "    \n",
    "    x_3 = Conv1D(filters=filter_size,kernel_size=kernel)(x_dep)\n",
    "    x_3 = GlobalMaxPool1D()(x_3)\n",
    "    kernel_dep.append(x_3)\n",
    "    \n",
    "conv_in1 = tensorflow.keras.layers.concatenate(kernel_stmt)\n",
    "conv_in1 = Dropout(0.6)(conv_in1)\n",
    "conv_in1 = Dense(128, activation='relu')(conv_in1)\n",
    "\n",
    "conv_in2 = tensorflow.keras.layers.concatenate(kernel_pos)\n",
    "conv_in2 = Dropout(0.6)(conv_in2)\n",
    "conv_in2 = Dense(128, activation='relu')(conv_in2)\n",
    "\n",
    "conv_in3 = tensorflow.keras.layers.concatenate(kernel_dep)\n",
    "conv_in3 = Dropout(0.6)(conv_in3)\n",
    "conv_in3 = Dense(128, activation='relu')(conv_in3)\n",
    "\n",
    "# meta data\n",
    "meta_input = Input(shape=(X_train_meta.shape[1],), name='aux_input')\n",
    "x_meta = Dense(64, activation='relu')(meta_input)\n",
    "\n",
    "if use_pos and use_meta:\n",
    "  if use_dep:\n",
    "    x = tensorflow.keras.layers.concatenate([conv_in1, conv_in2, conv_in3, x_meta])\n",
    "  else:\n",
    "    x = tensorflow.keras.layers.concatenate([conv_in1, conv_in2, x_meta])\n",
    "elif use_meta:\n",
    "  if use_dep:\n",
    "    x = tensorflow.keras.layers.concatenate([conv_in1, conv_in3, x_meta])\n",
    "  else:\n",
    "    x = tensorflow.keras.layers.concatenate([conv_in1, x_meta])\n",
    "elif use_pos:\n",
    "  if use_dep:\n",
    "    x = tensorflow.keras.layers.concatenate([conv_in1, conv_in2, conv_in3])\n",
    "  else:\n",
    "    x = tensorflow.keras.layers.concatenate([conv_in1, conv_in2])\n",
    "else:\n",
    "  if use_dep:\n",
    "    x = tensorflow.keras.layers.concatenate([conv_in1, conv_in3])\n",
    "  else:\n",
    "    x = conv_in1\n",
    "\n",
    "\n",
    "\n",
    "main_output = Dense(6, activation='softmax', name='main_output')(x)\n",
    "\n",
    "if use_pos and use_meta:\n",
    "  if use_dep:\n",
    "    model_cnn = Model(inputs=[statement_input, pos_input, dep_input, meta_input], outputs=[main_output])\n",
    "  else:\n",
    "    model_cnn = Model(inputs=[statement_input, pos_input, meta_input], outputs=[main_output])\n",
    "elif use_meta:\n",
    "  if use_dep:\n",
    "    model_cnn = Model(inputs=[statement_input, dep_input, meta_input], outputs=[main_output])\n",
    "  else:\n",
    "    model_cnn = Model(inputs=[statement_input, meta_input], outputs=[main_output])\n",
    "elif use_pos:\n",
    "  if use_dep:\n",
    "    model_cnn = Model(inputs=[statement_input, pos_input, dep_input], outputs=[main_output])\n",
    "  else:\n",
    "    model_cnn = Model(inputs=[statement_input, pos_input], outputs=[main_output])\n",
    "else:\n",
    "  if use_dep:\n",
    "    model_cnn = Model(inputs=[statement_input, dep_input], outputs=[main_output])\n",
    "  else:\n",
    "    model_cnn = Model(inputs=[statement_input], outputs=[main_output])\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "main_input (InputLayer)         [(None, 15)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dep_input (InputLayer)          [(None, 15)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 15, 100)      1240900     main_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 15, 10)       100         dep_input[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d (Conv1D)                 (None, 13, 128)      38528       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 13, 128)      38528       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)               (None, 13, 128)      38528       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 13, 128)      3968        embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)               (None, 13, 128)      3968        embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_8 (Conv1D)               (None, 13, 128)      3968        embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d (GlobalMax (None, 128)          0           conv1d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_3 (GlobalM (None, 128)          0           conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_6 (GlobalM (None, 128)          0           conv1d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 128)          0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_5 (GlobalM (None, 128)          0           conv1d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_8 (GlobalM (None, 128)          0           conv1d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 384)          0           global_max_pooling1d[0][0]       \n",
      "                                                                 global_max_pooling1d_3[0][0]     \n",
      "                                                                 global_max_pooling1d_6[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 384)          0           global_max_pooling1d_2[0][0]     \n",
      "                                                                 global_max_pooling1d_5[0][0]     \n",
      "                                                                 global_max_pooling1d_8[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 384)          0           concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 384)          0           concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "aux_input (InputLayer)          [(None, 83)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 128)          49280       dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 128)          49280       dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 64)           5376        aux_input[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 320)          0           dense[0][0]                      \n",
      "                                                                 dense_2[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "main_output (Dense)             (None, 6)            1926        concatenate_3[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 1,474,350\n",
      "Trainable params: 1,474,350\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print (model_cnn.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, name, use_pos=False, use_meta=False, use_dep=False):\n",
    "  sgd = optimizers.SGD(lr=0.025, clipvalue=0.3, nesterov=True)\n",
    "  adam = optimizers.Adam(lr=0.000075, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n",
    "  model.compile(optimizer=sgd,loss='categorical_crossentropy',metrics=['categorical_accuracy'])\n",
    "  tb = TensorBoard()\n",
    "  csv_logger = tensorflow.keras.callbacks.CSVLogger('training.log')\n",
    "  filepath= name+\"_weights_best.hdf5\"\n",
    "  checkpoint = tensorflow.keras.callbacks.ModelCheckpoint(filepath, monitor='val_categorical_accuracy', \n",
    "                                             verbose=1, save_best_only=True, mode='max')\n",
    "  if use_pos and use_meta:\n",
    "    if use_dep:\n",
    "      print('hi1')\n",
    "      model.fit(\n",
    "        {'main_input': X_train, 'pos_input': X_train_pos, \n",
    "         'aux_input': X_train_meta, 'dep_input': X_train_dep},\n",
    "        {'main_output': Y_train}, epochs = num_epochs, batch_size = batch_size,\n",
    "        validation_data = (\n",
    "            {'main_input': X_val, 'pos_input': X_val_pos, \n",
    "             'aux_input': X_val_meta, 'dep_input' : X_val_dep},\n",
    "            {'main_output': Y_val}\n",
    "        ), callbacks=[tb,csv_logger,checkpoint])\n",
    "    else:\n",
    "      print('hi2')\n",
    "      model.fit(\n",
    "        {'main_input': X_train, 'pos_input': X_train_pos, 'aux_input': X_train_meta},\n",
    "        {'main_output': Y_train}, epochs = num_epochs, batch_size = batch_size,\n",
    "        validation_data = ({'main_input': X_val, 'pos_input': X_val_pos, 'aux_input': X_val_meta},\n",
    "            {'main_output': Y_val}), callbacks=[tb,csv_logger,checkpoint])\n",
    "  elif use_meta:\n",
    "    if use_dep:\n",
    "      print('hi3')\n",
    "      model.fit(\n",
    "        {'main_input': X_train, 'aux_input': X_train_meta,'dep_input':X_train_dep},\n",
    "        {'main_output': Y_train}, epochs = num_epochs, batch_size = batch_size,\n",
    "        validation_data = (\n",
    "            {'main_input': X_val, 'aux_input': X_val_meta, 'dep_input': X_val_dep},\n",
    "            {'main_output': Y_val}\n",
    "        ), callbacks=[tb,csv_logger,checkpoint])\n",
    "    else:\n",
    "      model.fit(\n",
    "        {'main_input': X_train, 'aux_input': X_train_meta},\n",
    "        {'main_output': Y_train}, epochs = num_epochs, batch_size = batch_size,\n",
    "        validation_data = (\n",
    "            {'main_input': X_val, 'aux_input': X_val_meta},\n",
    "            {'main_output': Y_val}\n",
    "        ), callbacks=[tb,csv_logger,checkpoint])\n",
    "  elif use_pos:\n",
    "    if use_dep:\n",
    "      \n",
    "      model.fit(\n",
    "        {'main_input': X_train, 'pos_input': X_train_pos,'dep_input':X_train_dep},\n",
    "        {'main_output': Y_train}, epochs = num_epochs, batch_size = batch_size, validation_data = (\n",
    "            {'main_input': X_val, 'pos_input': X_val_pos, 'dep_input':X_val_dep},\n",
    "            {'main_output': Y_val}\n",
    "        ), callbacks=[tb,csv_logger,checkpoint])\n",
    "    else:\n",
    "      model.fit(\n",
    "        {'main_input': X_train, 'pos_input': X_train_pos},\n",
    "        {'main_output': Y_train}, epochs = num_epochs, batch_size = batch_size,\n",
    "        validation_data = (\n",
    "            {'main_input': X_val, 'pos_input': X_val_pos},\n",
    "            {'main_output': Y_val}\n",
    "        ), callbacks=[tb,csv_logger,checkpoint])\n",
    "  else:\n",
    "    if use_dep:\n",
    "      model.fit({'main_input': X_train,'dep_input':X_train_dep},\n",
    "        {'main_output': Y_train}, epochs = num_epochs, batch_size = batch_size,\n",
    "        validation_data = (\n",
    "            {'main_input': X_val, 'dep_input':X_val_dep},\n",
    "            {'main_output': Y_val}\n",
    "        ), callbacks=[tb,csv_logger,checkpoint])\n",
    "    else:\n",
    "      model.fit( {'main_input': X_train},{'main_output': Y_train}, epochs = num_epochs, batch_size = batch_size, \n",
    "                validation_data = ({'main_input': X_val},{'main_output': Y_val}), callbacks=[tb,csv_logger,checkpoint])\n",
    "    \n",
    "  \n",
    "  \n",
    "def evaluate(name, use_pos=False, use_meta=False, use_dep=False):\n",
    "  model1 = load_model(name+'_weights_best.hdf5')\n",
    "  if use_pos and use_meta:\n",
    "    if use_dep:\n",
    "      preds = model1.predict([X_test,X_test_pos, X_test_dep, X_test_meta], batch_size=batch_size, verbose=1)\n",
    "    else:\n",
    "      preds = model1.predict([X_test,X_test_pos, X_test_meta], batch_size=batch_size, verbose=1)\n",
    "  elif use_meta:\n",
    "    if use_dep:\n",
    "      preds = model1.predict([X_test, X_test_dep, X_test_meta], batch_size=batch_size, verbose=1)\n",
    "    else:\n",
    "      preds = model1.predict([X_test, X_test_meta], batch_size=batch_size, verbose=1)\n",
    "  elif use_pos:\n",
    "    if use_dep:\n",
    "      preds = model1.predict([X_test, X_test_pos, X_test_dep], batch_size=batch_size, verbose=1)\n",
    "    else:\n",
    "      preds = model1.predict([X_test, X_test_pos], batch_size=batch_size, verbose=1)\n",
    "  else:\n",
    "    if use_dep:\n",
    "      preds = model1.predict([X_test, X_test_dep], batch_size=batch_size, verbose=1)\n",
    "    else:\n",
    "      preds = model1.predict([X_test], batch_size=batch_size, verbose=1)\n",
    "    \n",
    "  false_worst = {}\n",
    "  true_best = {}\n",
    "  label_list = ['pants-fire','false','barely-true','half-true','mostly-true','true']\n",
    "\n",
    "  Y_test_gt = list(test_data['output'])\n",
    "  predictions = np.array([np.argmax(pred) for pred in preds])\n",
    "  \n",
    "  for p in range(len(preds)):\n",
    "    if np.argmax(preds[p])==0:\n",
    "      false_worst[p]=preds[p][0]\n",
    "    elif np.argmax(preds[p])==5:\n",
    "      true_best[p]=preds[p][5]\n",
    "      \n",
    "  print (len(predictions)==len(Y_test_gt))\n",
    "  correct = np.sum(predictions == Y_test_gt)\n",
    "  print (\"Correctly Predicted : \", correct,\"/\",len(Y_test_gt))\n",
    "  print (\"Accuracy : \", correct*100.0/len(Y_test_gt))\n",
    "  pickle.dump(predictions, open(name+'_predictions.p','wb'))\n",
    "  return (false_worst, true_best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hi3\n",
      "Train on 10240 samples, validate on 1284 samples\n",
      "Epoch 1/30\n",
      "   40/10240 [..............................] - ETA: 4:26WARNING:tensorflow:Can save best model only with val_categorical_accuracy available, skipping.\n"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": " indices[0,4] = 10 is not in [0, 10)\n\t [[node model/embedding_2/embedding_lookup (defined at <ipython-input-54-57c88bb5c904>:38) ]] [Op:__inference_distributed_function_20003]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node model/embedding_2/embedding_lookup:\n model/embedding_2/embedding_lookup/19444 (defined at c:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\contextlib.py:112)\n\nFunction call stack:\ndistributed_function\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-55-6d913c48a8e6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_cnn\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'cnn'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muse_pos\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muse_meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muse_dep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-54-57c88bb5c904>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(model, name, use_pos, use_meta, use_dep)\u001b[0m\n\u001b[0;32m     36\u001b[0m             \u001b[1;33m{\u001b[0m\u001b[1;34m'main_input'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mX_val\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'aux_input'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mX_val_meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'dep_input'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mX_val_dep\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     37\u001b[0m             \u001b[1;33m{\u001b[0m\u001b[1;34m'main_output'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mY_val\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 38\u001b[1;33m         ), callbacks=[tb,csv_logger,checkpoint])\n\u001b[0m\u001b[0;32m     39\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m       model.fit(\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[0;32m    817\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    818\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 819\u001b[1;33m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[0;32m    820\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    821\u001b[0m   def evaluate(self,\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[0;32m    340\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    341\u001b[0m                 \u001b[0mtraining_context\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtraining_context\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 342\u001b[1;33m                 total_epochs=epochs)\n\u001b[0m\u001b[0;32m    343\u001b[0m             \u001b[0mcbks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_logs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtraining_result\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    344\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2.py\u001b[0m in \u001b[0;36mrun_one_epoch\u001b[1;34m(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)\u001b[0m\n\u001b[0;32m    126\u001b[0m         step=step, mode=mode, size=current_batch_size) as batch_logs:\n\u001b[0;32m    127\u001b[0m       \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 128\u001b[1;33m         \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    129\u001b[0m       \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mStopIteration\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOutOfRangeError\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m         \u001b[1;31m# TODO(kaftan): File bug about tf function and errors.OutOfRangeError?\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2_utils.py\u001b[0m in \u001b[0;36mexecution_function\u001b[1;34m(input_fn)\u001b[0m\n\u001b[0;32m     96\u001b[0m     \u001b[1;31m# `numpy` translates Tensors to values in Eager mode.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     97\u001b[0m     return nest.map_structure(_non_none_constant_value,\n\u001b[1;32m---> 98\u001b[1;33m                               distributed_function(input_fn))\n\u001b[0m\u001b[0;32m     99\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    100\u001b[0m   \u001b[1;32mreturn\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    566\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    567\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 568\u001b[1;33m       \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    569\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    570\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    630\u001b[0m         \u001b[1;31m# Lifting succeeded, so variables are initialized and we can run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    631\u001b[0m         \u001b[1;31m# stateless function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 632\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    633\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    634\u001b[0m       \u001b[0mcanon_args\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcanon_kwds\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2361\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2362\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2363\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2364\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2365\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[1;34m(self, args, kwargs)\u001b[0m\n\u001b[0;32m   1609\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[0;32m   1610\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[1;32m-> 1611\u001b[1;33m         self.captured_inputs)\n\u001b[0m\u001b[0;32m   1612\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1613\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1690\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1691\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[1;32m-> 1692\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[0;32m   1693\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[0;32m   1694\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    543\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"executor_type\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"config_proto\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 545\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    546\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    547\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     65\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 67\u001b[1;33m     \u001b[0msix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     68\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     69\u001b[0m     keras_symbolic_tensors = [\n",
      "\u001b[1;32mc:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\six.py\u001b[0m in \u001b[0;36mraise_from\u001b[1;34m(value, from_value)\u001b[0m\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m:  indices[0,4] = 10 is not in [0, 10)\n\t [[node model/embedding_2/embedding_lookup (defined at <ipython-input-54-57c88bb5c904>:38) ]] [Op:__inference_distributed_function_20003]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node model/embedding_2/embedding_lookup:\n model/embedding_2/embedding_lookup/19444 (defined at c:\\users\\naik9\\appdata\\local\\programs\\python\\python37\\lib\\contextlib.py:112)\n\nFunction call stack:\ndistributed_function\n"
     ]
    }
   ],
   "source": [
    "train(model_cnn,'cnn', use_pos, use_meta, use_dep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(fw, tb) = evaluate('cnn', use_pos, use_meta, use_dep)\n",
    "print_best_false_true_predicted(fw, tb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}