Home_Work_1and2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class XML2DataFrame:\n",
    "    def __init__(self, xml_path):\n",
    "        xml_data = open(xml_path)\n",
    "        self.root = ET.XML(xml_data.read())[1]\n",
    "\n",
    "    def parse_root(self, root):\n",
    "        return [self.parse_element(child) for child in iter(root)]\n",
    "\n",
    "    def parse_element(self, element, parsed=None):\n",
    "        if parsed is None:\n",
    "            parsed = dict()\n",
    "        for key in element.keys():\n",
    "            parsed[key] = element.attrib.get(key)\n",
    "        if element.text:\n",
    "            parsed[element.attrib[\"name\"]] = None if element.text == \"NULL\" else element.text\n",
    "        for child in list(element):\n",
    "            self.parse_element(child, parsed)\n",
    "        return parsed\n",
    "\n",
    "    def process_data(self):\n",
    "        structure_data = self.parse_root(self.root)\n",
    "        return pd.DataFrame(structure_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tkk = XML2DataFrame(\"tkk_train_2016.xml\").process_data().fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tkk_texts = train_tkk[\"text\"].values\n",
    "train_tkk_labels = train_tkk[[\"beeline\", \"komstar\", \"mts\", \"rostelecom\", \"skylink\", \"tele2\"]].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['@mkomov Максим, Вашем письмо мы получили. Наши сотрудники свяжутся с Вами завтра и направят запрос инженерам для проверки. #билайн',\n",
       "       '«Мегафон» стал владельцем 50% акций «Евросети»',\n",
       "       'RT @fuckkiev: “@EvaKobb: МТС Россия прислала жителям Херсонщины сообщения, в которых обозвала украинцев фашистами? http://t.co/RbSesXlOUZ” …',\n",
       "       ...,\n",
       "       'RT @Olympialeigh: У МТС проблемы со связью из-за замены оборудования',\n",
       "       'RT @ManisaMerkez45: У МТС проблемы со связью из-за замены оборудования',\n",
       "       'RT @MertKaraoban: У МТС проблемы со связью из-за замены оборудования'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_tkk_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tkk = XML2DataFrame(\"tkk_test_etalon.xml\").process_data().fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tkk_texts = test_tkk[\"text\"].values\n",
    "test_tkk_labels = test_tkk[[\"beeline\", \"komstar\", \"mts\", \"rostelecom\", \"skylink\", \"tele2\"]].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_labels = []\n",
    "for string in train_tkk_labels:\n",
    "    new_labels.append(sum([int(val) for val in string]))\n",
    "    if new_labels[-1]<-1:\n",
    "        new_labels[-1]=-1\n",
    "    elif new_labels[-1]>1:\n",
    "        new_labels[-1]=1\n",
    "train_tkk_labels = new_labels\n",
    "\n",
    "new_labels = []\n",
    "for string in test_tkk_labels:\n",
    "    new_labels.append(sum([int(val) for val in string]))\n",
    "    if new_labels[-1]<-1:\n",
    "        new_labels[-1]=-1\n",
    "    elif new_labels[-1]>1:\n",
    "        new_labels[-1]=1\n",
    "test_tkk_labels=new_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vectorizer = TfidfVectorizer(min_df=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = vectorizer.fit(train_tkk_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(n_jobs=7,random_state=42,verbose=1,warm_start=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:1296: UserWarning: 'n_jobs' > 1 does not have any effect when 'solver' is set to 'liblinear'. Got 'n_jobs' = 7.\n",
      "  \" = {}.\".format(effective_n_jobs(self.n_jobs)))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn', n_jobs=7,\n",
       "          penalty='l2', random_state=42, solver='warn', tol=0.0001,\n",
       "          verbose=1, warm_start=True)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.fit(tfidf.transform(train_tkk_texts), train_tkk_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = list(lr.predict(tfidf.transform(test_tkk_texts)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4542176544883651\n",
      "0.654650645304851\n"
     ]
    }
   ],
   "source": [
    "# With no regularization except L2\n",
    "print(f1_score(test_tkk_labels,pred, average='macro'))\n",
    "print(f1_score(test_tkk_labels,pred, average='micro'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tkk_tfidf = tfidf.transform(train_tkk_texts).toarray()\n",
    "test_tkk_tfidf = tfidf.transform(test_tkk_texts).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tkk_tfidf_reg = np.array([(string-string.mean())/string.std() for string in train_tkk_tfidf])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tkk_tfidf_reg = np.array([(string-string.mean())/string.std() for string in test_tkk_tfidf])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(random_state=42,verbose=1,warm_start=True,max_iter=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=50, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=42, solver='warn',\n",
       "          tol=0.0001, verbose=1, warm_start=True)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.fit(train_tkk_tfidf_reg, train_tkk_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_reg = list(lr.predict(test_tkk_tfidf_reg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5266087980476643\n",
      "0.6444147752558967\n"
     ]
    }
   ],
   "source": [
    "print(f1_score(test_tkk_labels,pred_reg,average='macro'))\n",
    "print(f1_score(test_tkk_labels,pred_reg,average='micro'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best results at SentiRueval2016: <br>\n",
    "F1_macro = 0.5493 <br>\n",
    "F1_micro = 0.6822"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Banks analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_banks = XML2DataFrame('bank_train_2016.xml').process_data().fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_banks = XML2DataFrame('banks_test_etalon.xml').process_data().fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_banks_texts = train_banks['text'].values\n",
    "train_banks_labels = train_banks[[\n",
    "    \"sberbank\", \"vtb\", \"gazprom\", \"alfabank\", \"bankmoskvy\", \"raiffeisen\",\n",
    "    \"uralsib\", \"rshb\"\n",
    "]].values\n",
    "new_labels = []\n",
    "for string in train_banks_labels:\n",
    "    new_labels.append(sum([int(val) for val in string]))\n",
    "    if new_labels[-1]<-1:\n",
    "        new_labels[-1]=-1\n",
    "    elif new_labels[-1]>1:\n",
    "        new_labels[-1]=1\n",
    "train_banks_labels = new_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_banks_texts = test_banks['text'].values\n",
    "test_banks_labels = test_banks[[\n",
    "    \"sberbank\", \"vtb\", \"gazprom\", \"alfabank\", \"bankmoskvy\", \"raiffeisen\",\n",
    "    \"uralsib\", \"rshb\"\n",
    "]].values\n",
    "new_labels = []\n",
    "for string in test_banks_labels:\n",
    "    new_labels.append(sum([int(val) for val in string]))\n",
    "    if new_labels[-1]<-1:\n",
    "        new_labels[-1]=-1\n",
    "    elif new_labels[-1]>1:\n",
    "        new_labels[-1]=1\n",
    "test_banks_labels = new_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(n_jobs=3,random_state=42,verbose=1,warm_start=True,max_iter=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_banks = vectorizer.fit(train_banks_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_banks_tfidf = tfidf_banks.transform(train_banks_texts).toarray()\n",
    "test_banks_tfidf = tfidf_banks.transform(test_banks_texts).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:1296: UserWarning: 'n_jobs' > 1 does not have any effect when 'solver' is set to 'liblinear'. Got 'n_jobs' = 3.\n",
      "  \" = {}.\".format(effective_n_jobs(self.n_jobs)))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=50, multi_class='warn', n_jobs=3,\n",
       "          penalty='l2', random_state=42, solver='warn', tol=0.0001,\n",
       "          verbose=1, warm_start=True)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.fit(train_banks_tfidf,train_banks_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Banks results with no regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "banks_pred = list(lr.predict(test_banks_tfidf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4979622045828058\n",
      "0.73407787503773\n"
     ]
    }
   ],
   "source": [
    "print(f1_score(banks_pred,test_banks_labels,average='macro'))\n",
    "print(f1_score(banks_pred,test_banks_labels,average='micro'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best results at SentiRueval2016: <br>\n",
    "F1_macro = 0.5252 <br>\n",
    "F1_micro = 0.5881"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regularization for banks tf/idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_banks_tfidf_reg = np.array([(string-string.mean())/string.std() for string in train_banks_tfidf])\n",
    "test_banks_tfidf_reg = np.array([(string-string.mean())/string.std() for string in test_banks_tfidf])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(n_jobs=3,random_state=42,verbose=1,warm_start=True,max_iter=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/python/venv3.6/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:1228: UserWarning: 'n_jobs' > 1 does not have any effect when 'solver' is set to 'liblinear'. Got 'n_jobs' = 3.\n",
      "  \" = {}.\".format(self.n_jobs))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=50, multi_class='ovr', n_jobs=3,\n",
       "          penalty='l2', random_state=42, solver='liblinear', tol=0.0001,\n",
       "          verbose=1, warm_start=True)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.fit(train_banks_tfidf_reg,train_banks_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction for banks with regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "banks_pred_reg = list(lr.predict(test_banks_tfidf_reg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.29355505573876134\n",
      "0.7214005433142169\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/python/venv3.6/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print(f1_score(banks_pred_reg,test_banks_labels,average='macro'))\n",
    "print(f1_score(banks_pred_reg,test_banks_labels,average='micro'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best results at SentiRueval2016: <br>\n",
    "F1_macro = 0.5252 <br>\n",
    "F1_micro = 0.5881"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(n_jobs=3,random_state=42,verbose=1,warm_start=True,max_iter=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using all train tweets for both banks and tkk with logreg and no regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_train_labels = train_banks_labels\n",
    "total_train_labels.extend(train_tkk_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_train_texts = []\n",
    "total_train_texts.extend(train_banks_texts)\n",
    "total_train_texts.extend(train_tkk_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_tfidf = vectorizer.fit(total_train_texts)\n",
    "total_train_tfidf = total_tfidf.transform(total_train_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:1296: UserWarning: 'n_jobs' > 1 does not have any effect when 'solver' is set to 'liblinear'. Got 'n_jobs' = 3.\n",
      "  \" = {}.\".format(effective_n_jobs(self.n_jobs)))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=50, multi_class='warn', n_jobs=3,\n",
       "          penalty='l2', random_state=42, solver='warn', tol=0.0001,\n",
       "          verbose=1, warm_start=True)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.fit(total_train_tfidf,total_train_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results for test_banks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot_pred = list(lr.predict(total_tfidf.transform(test_banks_texts)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5382323874148384\n",
      "0.7524901901599759\n"
     ]
    }
   ],
   "source": [
    "print(f1_score(tot_pred,test_banks_labels,average='macro'))\n",
    "print(f1_score(tot_pred,test_banks_labels,average='micro'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best results at SentiRueval2016: <br>\n",
    "F1_macro = 0.5252 <br>\n",
    "F1_micro = 0.5881"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results for test_tkk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot_pred = list(lr.predict(total_tfidf.transform(test_tkk_texts)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4802356659930958\n",
      "0.6653315531820204\n"
     ]
    }
   ],
   "source": [
    "print(f1_score(tot_pred,test_tkk_labels,average='macro'))\n",
    "print(f1_score(tot_pred,test_tkk_labels,average='micro'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best results at SentiRueval2016: <br>\n",
    "F1_macro = 0.5493 <br>\n",
    "F1_micro = 0.6813"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN aproach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "WARNING (theano.tensor.blas): Using NumPy C-API based implementation for BLAS functions.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ['KERAS_BACKEND'] = 'theano'\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv1D, Embedding, Dropout, Dense, GlobalAveragePooling1D, MaxPooling1D, Softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import FastText"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "wvs = np.array([lang_model.wv[sent] for sent in prepared_for_embed])\n",
    "size= 50\n",
    "padded = pad_sequences(wvs,padding='post', value=np.full(size,0.0))\n",
    "\n",
    "from pandas import get_dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_banks_wvs = []\n",
    "for sent in test_banks_texts:\n",
    "    prep_sent = []\n",
    "    for word in sent.split():\n",
    "        try:\n",
    "            prep_sent.append(lang_model.wv[word])\n",
    "        except KeyError:\n",
    "            pass\n",
    "    test_banks_wvs.append(np.array(prep_sent))\n",
    "test_banks_wvs = np.array(test_banks_wvs)\n",
    "\n",
    "test_banks_padded = pad_sequences(test_banks_wvs,padding='post', maxlen=30, value=np.full(size,0.0))\n",
    "\n",
    "test_tkk_wvs = []\n",
    "for sent in test_tkk_texts:\n",
    "    prep_sent = []\n",
    "    for word in sent.split():\n",
    "        try:\n",
    "            prep_sent.append(lang_model.wv[word])\n",
    "        except KeyError:\n",
    "            pass\n",
    "    test_tkk_wvs.append(np.array(prep_sent))\n",
    "test_tkk_wvs = np.array(test_tkk_wvs)\n",
    "\n",
    "test_tkk_padded = pad_sequences(test_tkk_wvs,padding='post', maxlen=30, value=np.full(size,0.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f1_macro_from_dummies(pred,gt, average='macro'):\n",
    "    gt_val = [np.argmax(string)-1 for string in gt]\n",
    "    pred_val = [np.argmax(string)-1 for string in pred]\n",
    "    return mean(np.array([f1_score(gt_val, pred_val, average=average)]))\n",
    "def f1_micro_from_dummies(pred,gt, average='micro'):\n",
    "    gt_val = [np.argmax(string)-1 for string in gt]\n",
    "    pred_val = [np.argmax(string)-1 for string in pred]\n",
    "    return mean(np.array([f1_score(gt_val, pred_val, average=average)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "size= 50\n",
    "lang_model = FastText(min_count=1,window=5,workers=7,size=size)\n",
    "prepared_for_embed = [sent.split() for sent in total_train_texts]\n",
    "lang_model.build_vocab(prepared_for_embed)\n",
    "lang_model.train(prepared_for_embed, total_examples=lang_model.corpus_count, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "seq_length=30\n",
    "model.add(Conv1D(64, 10, activation='relu',input_shape=(seq_length, size)))\n",
    "model.add(MaxPooling1D(10))\n",
    "model.add(GlobalAveragePooling1D())\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(3, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adagrad',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 18035 samples, validate on 3313 samples\n",
      "Epoch 1/20\n",
      "18035/18035 [==============================] - 22s 1ms/step - loss: 0.5988 - acc: 0.7378 - val_loss: 0.8979 - val_acc: 0.5638\n",
      "Epoch 2/20\n",
      "18035/18035 [==============================] - 22s 1ms/step - loss: 0.5974 - acc: 0.7407 - val_loss: 0.8712 - val_acc: 0.5744\n",
      "Epoch 3/20\n",
      "18035/18035 [==============================] - 22s 1ms/step - loss: 0.5930 - acc: 0.7393 - val_loss: 0.8896 - val_acc: 0.5729\n",
      "Epoch 4/20\n",
      "18035/18035 [==============================] - 23s 1ms/step - loss: 0.5912 - acc: 0.7412 - val_loss: 0.8976 - val_acc: 0.5626\n",
      "Epoch 5/20\n",
      " 4096/18035 [=====>........................] - ETA: 15s - loss: 0.5855 - acc: 0.7385"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-79-15623e3b4446>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m           validation_data=(test_banks_padded,\n\u001b[1;32m      3\u001b[0m                            get_dummies(test_banks_labels).values),\n\u001b[0;32m----> 4\u001b[0;31m           batch_size=128, epochs=20)\n\u001b[0m",
      "\u001b[0;32m/mnt/c/DataScience/venv/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/mnt/c/DataScience/venv/lib/python3.7/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/c/DataScience/venv/lib/python3.7/site-packages/keras/backend/theano_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1386\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1387\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1388\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1389\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1390\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/c/DataScience/venv/lib/python3.7/site-packages/theano/compile/function_module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    901\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    902\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 903\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    904\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    905\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/c/DataScience/venv/lib/python3.7/site-packages/theano/gof/op.py\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n)\u001b[0m\n\u001b[1;32m    889\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mparams\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNoParams\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    890\u001b[0m             \u001b[0;31m# default arguments are stored in the closure of `rval`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 891\u001b[0;31m             \u001b[0;32mdef\u001b[0m \u001b[0mrval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode_input_storage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode_output_storage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    892\u001b[0m                 \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    893\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(padded, get_dummies(total_train_labels).values,\n",
    "          validation_data=(test_banks_padded,\n",
    "                           get_dummies(test_banks_labels).values),\n",
    "          batch_size=128, epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO (theano.gof.compilelock): Refreshing lock /home/zubkov/.theano/compiledir_Linux-4.4--Microsoft-x86_64-with-Ubuntu-16.04-xenial-x86_64-3.7.0-64/lock_dir/lock\n"
     ]
    }
   ],
   "source": [
    "cnn_pred = model.predict(test_banks_padded)\n",
    "cnn_pred_val = [np.argmax(string)-1 for string in cnn_pred]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3866006424419824\n",
      "0.5759130697253245\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/DataScience/venv/lib/python3.7/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print(f1_score(cnn_pred_val,test_banks_labels, average='macro'))\n",
    "print(f1_score(cnn_pred_val,test_banks_labels, average='micro'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn_pred = model.predict(test_tkk_padded)\n",
    "cnn_pred_val = [np.argmax(string)-1 for string in cnn_pred]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3588088335999437\n",
      "0.5331553182020472\n"
     ]
    }
   ],
   "source": [
    "print(f1_score(cnn_pred_val,test_tkk_labels, average='macro'))\n",
    "print(f1_score(cnn_pred_val,test_tkk_labels, average='micro'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f1(y_true, y_pred):\n",
    "    def recall(y_true, y_pred):\n",
    "        \"\"\"Recall metric.\n",
    "\n",
    "        Only computes a batch-wise average of recall.\n",
    "\n",
    "        Computes the recall, a metric for multi-label classification of\n",
    "        how many relevant items are selected.\n",
    "        \"\"\"\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "        recall = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall\n",
    "\n",
    "    def precision(y_true, y_pred):\n",
    "        \"\"\"Precision metric.\n",
    "\n",
    "        Only computes a batch-wise average of precision.\n",
    "\n",
    "        Computes the precision, a metric for multi-label classification of\n",
    "        how many selected items are relevant.\n",
    "        \"\"\"\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "        precision = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision\n",
    "    precision = precision(y_true, y_pred)\n",
    "    recall = recall(y_true, y_pred)\n",
    "    return 2*((precision*recall)/(precision+recall+K.epsilon()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f1_loss(y_true,y_pred):\n",
    "    def recall(y_true, y_pred):\n",
    "        \"\"\"Recall metric.\n",
    "\n",
    "        Only computes a batch-wise average of recall.\n",
    "\n",
    "        Computes the recall, a metric for multi-label classification of\n",
    "        how many relevant items are selected.\n",
    "        \"\"\"\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "        recall = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall\n",
    "\n",
    "    def precision(y_true, y_pred):\n",
    "        \"\"\"Precision metric.\n",
    "\n",
    "        Only computes a batch-wise average of precision.\n",
    "\n",
    "        Computes the precision, a metric for multi-label classification of\n",
    "        how many selected items are relevant.\n",
    "        \"\"\"\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "        precision = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision\n",
    "    precision = precision(y_true, y_pred)\n",
    "    recall = recall(y_true, y_pred)\n",
    "    return (precision+recall+K.epsilon())/(2*((precision*recall)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import FastText"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrained = FastText.load('araneum_none_fasttextskipgram_300_5_2018/araneum_none_fasttextskipgram_300_5_2018.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "seq_length=30\n",
    "model.add(Conv1D(64, 10, activation='relu',input_shape=(seq_length, pretrained.vector_size)))\n",
    "model.add(MaxPooling1D(10))\n",
    "model.add(GlobalAveragePooling1D())\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(3, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adagrad',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "wvs = []\n",
    "for sent in prepared_for_embed:\n",
    "    sent_vec = []\n",
    "    for word in sent:\n",
    "        try:\n",
    "            sent_vec.append(pretrained.wv[word])\n",
    "        except KeyError:\n",
    "            pass\n",
    "    wvs.append(np.array(sent_vec))\n",
    "wvs = np.array(wvs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "padded = pad_sequences(wvs, padding='post', maxlen=30, value=np.full(pretrained.vector_size,0.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang_model = pretrained\n",
    "test_banks_wvs = []\n",
    "for sent in test_banks_texts:\n",
    "    prep_sent = []\n",
    "    for word in sent.split():\n",
    "        try:\n",
    "            prep_sent.append(lang_model.wv[word])\n",
    "        except KeyError:\n",
    "            pass\n",
    "    test_banks_wvs.append(np.array(prep_sent))\n",
    "test_banks_wvs = np.array(test_banks_wvs)\n",
    "\n",
    "test_banks_padded = pad_sequences(test_banks_wvs,padding='post', maxlen=30, value=np.full(lang_model.vector_size,0.0))\n",
    "\n",
    "test_tkk_wvs = []\n",
    "for sent in test_tkk_texts:\n",
    "    prep_sent = []\n",
    "    for word in sent.split():\n",
    "        try:\n",
    "            prep_sent.append(lang_model.wv[word])\n",
    "        except KeyError:\n",
    "            pass\n",
    "    test_tkk_wvs.append(np.array(prep_sent))\n",
    "test_tkk_wvs = np.array(test_tkk_wvs)\n",
    "\n",
    "test_tkk_padded = pad_sequences(test_tkk_wvs,padding='post', maxlen=30, value=np.full(lang_model.vector_size,0.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 18035 samples, validate on 3313 samples\n",
      "Epoch 1/20\n",
      "18035/18035 [==============================] - 118s 7ms/step - loss: 0.8468 - acc: 0.6921 - val_loss: 0.7823 - val_acc: 0.6954\n",
      "Epoch 2/20\n",
      "18035/18035 [==============================] - 119s 7ms/step - loss: 0.7706 - acc: 0.7029 - val_loss: 0.7715 - val_acc: 0.7027\n",
      "Epoch 3/20\n",
      "18035/18035 [==============================] - 115s 6ms/step - loss: 0.7545 - acc: 0.7072 - val_loss: 0.7716 - val_acc: 0.7093\n",
      "Epoch 4/20\n",
      "18035/18035 [==============================] - 117s 6ms/step - loss: 0.7483 - acc: 0.7102 - val_loss: 0.7687 - val_acc: 0.7096\n",
      "Epoch 5/20\n",
      " 7040/18035 [==========>...................] - ETA: 1:10 - loss: 0.7356 - acc: 0.7202"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-125-15623e3b4446>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m           validation_data=(test_banks_padded,\n\u001b[1;32m      3\u001b[0m                            get_dummies(test_banks_labels).values),\n\u001b[0;32m----> 4\u001b[0;31m           batch_size=128, epochs=20)\n\u001b[0m",
      "\u001b[0;32m/mnt/c/DataScience/venv/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/mnt/c/DataScience/venv/lib/python3.7/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/c/DataScience/venv/lib/python3.7/site-packages/keras/backend/theano_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1386\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1387\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1388\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1389\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1390\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/c/DataScience/venv/lib/python3.7/site-packages/theano/compile/function_module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    901\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    902\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 903\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    904\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    905\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/c/DataScience/venv/lib/python3.7/site-packages/theano/gof/op.py\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n)\u001b[0m\n\u001b[1;32m    889\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mparams\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNoParams\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    890\u001b[0m             \u001b[0;31m# default arguments are stored in the closure of `rval`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 891\u001b[0;31m             \u001b[0;32mdef\u001b[0m \u001b[0mrval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode_input_storage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode_output_storage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    892\u001b[0m                 \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    893\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(padded, get_dummies(total_train_labels).values,\n",
    "          validation_data=(test_banks_padded,\n",
    "                           get_dummies(test_banks_labels).values),\n",
    "          batch_size=128, epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18035, 30, 300)"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}