neuroText.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "from future import standard_library\n",
    "import time\n",
    "import sys\n",
    "import os\n",
    "import wget\n",
    "import re\n",
    "from ufal.udpipe import Model, Pipeline\n",
    "import numpy as np\n",
    "from scipy.cluster import *\n",
    "import word2vec\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "\n",
    "from IPython.display import clear_output\n",
    "def log_progress(sequence, every=10):\n",
    "    from ipywidgets import IntProgress\n",
    "    from IPython.display import display\n",
    "\n",
    "    progress = IntProgress(min=0, max=len(sequence), value=0)\n",
    "    display(progress)\n",
    "    \n",
    "    for index, record in enumerate(sequence):\n",
    "        if index % every == 0:\n",
    "            progress.value = index\n",
    "        yield record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "FILEmodelForInfinitive = Model.load('udpipe_syntagrus.model')\n",
    "#model = word2vec.load('dataForNeurotext/withOutTypeModel.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 0. 0. ... 0. 0. 0.]\n",
      "CPU times: user 230 ms, sys: 17.2 ms, total: 247 ms\n",
      "Wall time: 264 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "class wordsClass (object):\n",
    "    model = ''\n",
    "    vectorOfStr = np.zeros(2000)\n",
    "    centroids = torch.zeros(2000,300)\n",
    "    ClasterOfWord = {}\n",
    "    nomberKnownOfWords = 0\n",
    "    nomberUNKnownOfWords = 0\n",
    "    modelForInfinitive = ''\n",
    "\n",
    "    def loadCentroidsFromTXT (self, source):\n",
    "        self.centroids = torch.from_numpy(np.loadtxt(source, delimiter=' ', dtype=float))\n",
    "        self.centroids = torch.reshape(self.centroids, (2000, 300)).cuda()\n",
    "        \n",
    "    def saveCentroidsToTXT (self, source):\n",
    "        f = open (source, 'w')\n",
    "        for i in range (2000):\n",
    "            for j in range (300):\n",
    "                f.write(str(self.centroids[i][j]) + \" \")\n",
    "        f.close()\n",
    "        \n",
    "    def saveDict (self, source):\n",
    "        f = open(source, 'w')\n",
    "        for k, v in self.ClasterOfWord.items():\n",
    "            f.write(str(k) + \" \" + str(v) + \"\\n\")\n",
    "        f.close()\n",
    "            \n",
    "    def loadDict (self, source):\n",
    "        f = open(source, 'r')\n",
    "        for strings in f:\n",
    "            strings = strings.split(' ')\n",
    "            self.ClasterOfWord.update({strings[0]: int(strings[1])})\n",
    "        f.close()\n",
    "        \n",
    "    def generateCentroids (self, nomberOfClasters, NomberOfIter):\n",
    "        #scipy.cluster.vq.kmeans2(model.vectors, 2000, iter=100)\n",
    "        self.centroids = vq.kmeans(self.model.vectors, nomberOfClasters, iter=NomberOfIter)[0]\n",
    "        \n",
    "    def generateDict (self, source): #источник = txt для model\n",
    "        f = open (source, 'r')\n",
    "        n=0\n",
    "        words = []\n",
    "        for line in f:\n",
    "            line = line.split()\n",
    "            words.append(line[0])\n",
    "            n = n+1\n",
    "        print (words[1])\n",
    "        f.close()\n",
    "        for word in log_progress(words, every=100):\n",
    "            distToCentroids = torch.sum(((self.centroids - self.model[word])**2),1)\n",
    "            relationID = int(torch.argmin(distToCentroids, 0))\n",
    "            ClasterOfWord.update({word: relationID})\n",
    "    \n",
    "    def distanceBetween2words (self, word1, word2):\n",
    "        word1 = word1.lower()\n",
    "        word2 = word2.lower()\n",
    "        try:\n",
    "            len = model.distance(tag_ud(text=word1), tag_ud(text=word2))\n",
    "        except Exception:\n",
    "            try:\n",
    "                len =  model.distance(tag_ud(text=word1.capitalize()), tag_ud(text=word2))\n",
    "            except Exception:\n",
    "                try:\n",
    "                    len =  model.distance(tag_ud(text=word1), tag_ud(text=word2.capitalize()))\n",
    "                except Exception:\n",
    "                    len = model.distance(tag_ud(text=word1.capitalize()), tag_ud(text=word2.capitalize()))\n",
    "        return len\n",
    "    \n",
    "    def coordinate (self, word):\n",
    "        word = word.lower()\n",
    "        try:\n",
    "            coord = self.model[tag_ud(text=word)]\n",
    "        except Exception:\n",
    "            try:\n",
    "                coord =  self.model[tag_ud(text=word.capitalize())]\n",
    "            except Exception:\n",
    "                coord = \"error\"\n",
    "        return torch.from_numpy (coord)\n",
    "    \n",
    "    def getKey(self, value):\n",
    "        for k, v in self.ClasterOfWord.items():\n",
    "            if int(v) == value:\n",
    "                print (k)\n",
    "        return k\n",
    "    \n",
    "    def addNewWordInClaster (self, newWord, baseWord):\n",
    "        self.ClasterOfWord.update({newWord: self.ClasterOfWord[baseWord]})\n",
    "    \n",
    "    def tryToUpgradeSelfWord2vec (self, word):\n",
    "        try:\n",
    "            #print (word)\n",
    "            relationID = self.ClasterOfWord[word.lower()]\n",
    "            self.vectorOfStr[relationID] = self.vectorOfStr[relationID] + 1\n",
    "            self.nomberKnownOfWords = self.nomberKnownOfWords + 1\n",
    "            #print (relationID)\n",
    "        except Exception:\n",
    "            if not re.search(r'[\\W]', word):\n",
    "                #print(self.wordToInf(text=word))\n",
    "                relationID = self.ClasterOfWord[self.wordToInf(text=word)]\n",
    "                self.vectorOfStr[relationID] = self.vectorOfStr[relationID] + 1\n",
    "                self.nomberKnownOfWords = self.nomberKnownOfWords + 1\n",
    "                #print (relationID)\n",
    "            else: \n",
    "                raise\n",
    "                \n",
    "            \n",
    "    def text2Vec (self, text):\n",
    "        self.vectorOfStr = np.zeros(2000)\n",
    "        words = re.findall(r'[0-9A-Za-zА-Яа-я-.]+', text)\n",
    "        for word in words:\n",
    "            try :\n",
    "                self.tryToUpgradeSelfWord2vec(word)\n",
    "            except Exception:\n",
    "                for singleWord in re.findall(r\"[\\w']+\", word):\n",
    "                    try: \n",
    "                        self.tryToUpgradeSelfWord2vec(singleWord)\n",
    "                    except Exception:\n",
    "                        self.nomberUNKnownOfWords = self.nomberUNKnownOfWords + 1\n",
    "                        #print (\"the word is new: \" + singleWord)\n",
    "        self.vectorOfStr = self.vectorOfStr / self.nomberKnownOfWords    \n",
    "        return self.vectorOfStr\n",
    "    \n",
    "    def wordToInf(self, text):\n",
    "        process_pipeline = Pipeline(self.modelForInfinitive, 'tokenize', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu')\n",
    "        wordInfo = process_pipeline.process(text).split('\\n')[4].split('\\t')\n",
    "        if (wordInfo[3] == 'NUM'):\n",
    "            return ('_NUM_' + ('x' * len(wordInfo[2])))\n",
    "        else:\n",
    "            return wordInfo[2]\n",
    "        \n",
    "#проработать встречи ::\n",
    "#реализовать работу с реляционной БД (?):\n",
    "#(текст->оценка принадлежности вектор->оценка принадлежности)\n",
    "#поддержка новый слов через сервис\n",
    "\n",
    "wordsClass = wordsClass()\n",
    "wordsClass.modelForInfinitive = FILEmodelForInfinitive\n",
    "wordsClass.loadDict('correct_dictionary.txt')\n",
    "\n",
    "#wordsClass.text2Vec(\"рaссказывать\")\n",
    "#wordsClass.ClasterOfWord['Alibaba']\n",
    "#wordsClass.saveDict('dict words has clasters.txt')\n",
    "\n",
    "#wordsClass.text2Vec(\"Китайский интернет-гигант Alibaba меняет бизнес-модель торговой площадки AliExpress, дав возможность продавцам из России, а также Турции, Италии и Испании продавать на ней свои товары. Об этом сообщает «Интерфакс».\")\n",
    "print (wordsClass.vectorOfStr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "NOfData = 1848\n",
    "input_size = 1929\n",
    "num_classes = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "relationTranslate = {\n",
    "    0: 'авто/мото',\n",
    "    1: 'активный отдых',\n",
    "    2: 'бизнес',\n",
    "    3: 'домашние животные',\n",
    "    4: 'здоровье',\n",
    "    5: 'знакомство и общение',\n",
    "    6: 'игры',\n",
    "    7: 'ИТ (компьютеры и софт)',\n",
    "    8: 'кино',\n",
    "    9: 'красота и мода',\n",
    "    10: 'кулинария',\n",
    "    11: 'культура и искусство',\n",
    "    12: 'литература',\n",
    "    13: 'мобильная связь и интернет',\n",
    "    14: 'музыка',\n",
    "    15: 'наука и техника',\n",
    "    16: 'недвижимость',\n",
    "    17: 'новости и СМИ',\n",
    "    18: 'безопасность',\n",
    "    19: 'образование',\n",
    "    20: 'обустройство и ремонт',\n",
    "    21: 'политика',\n",
    "    22: 'продукты питания',\n",
    "    23: 'промышленность',\n",
    "    24: 'путешествия',\n",
    "    25: 'работа',\n",
    "    26: 'развлечения',\n",
    "    27: 'религия',\n",
    "    28: 'дом и семья',\n",
    "    29: 'спорт',\n",
    "    30: 'страхование',\n",
    "    31: 'телевидение',\n",
    "    32: 'товары и услуги',\n",
    "    33: 'увлечения и хобби',\n",
    "    34: 'финансы',\n",
    "    35: 'фото',\n",
    "    36: 'эзотерика',\n",
    "    37: 'электроника и бытовая техника',\n",
    "    38: 'эротика',\n",
    "    39: 'юмор',\n",
    "    40: 'общество, гуманитарные науки',\n",
    "    41: 'дизайн и графика'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "badIndexes = np.zeros(2000)\n",
    "idexes = [58, 81,130,150,178,259,291,301,321,330,335,365,373,418,453,470,478,494,543,602,629,645,666,690,725,\n",
    "746,770,775,821,825,826,827,844,855,939,942,945,968,1011,1015,1040,1052,1083,1095,1111,1132,1205,1210,1302,\n",
    "1305,1314,1366,1371,1382,1393,1497,1511,1531,1535,1569,1571,1613,1649,1727,1740,1766,1807,1823,1837,1900,1914]\n",
    "for inde in idexes:\n",
    "    badIndexes[inde] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 30.5 s, sys: 93.3 ms, total: 30.6 s\n",
      "Wall time: 30.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#загрузка векторов и формирование list[vecOfText, vecOfRelation]\n",
    "\n",
    "vecOfTexts = torch.zeros(input_size)\n",
    "vecOfResult = torch.zeros(num_classes)\n",
    "strings = open('data.txt' ,'r').read().split ('\\n')\n",
    "vec = []\n",
    "for i  in range (NOfData):\n",
    "    i = 0\n",
    "    j = 0\n",
    "    k = 0\n",
    "    string = strings[i].split('|')\n",
    "    \n",
    "    for coordOfText in string[0].split('~'):\n",
    "        vecOfTexts[i] = 10000 * float(coordOfText)\n",
    "        i = i + 1\n",
    "\n",
    "    for coordOfResult in string[1].split('~'):\n",
    "        vecOfResult[j] = float(coordOfResult)\n",
    "        j = j + 1\n",
    "\n",
    "    vec.append([vecOfTexts, vecOfResult])\n",
    "    k = k + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#разделене на тренировочные и тестовые данные\n",
    "vec_train = []\n",
    "vec_test = []\n",
    "for i in range (NOfData):\n",
    "    if (i < 1700):\n",
    "        vec_train.append(vec [i])\n",
    "    else:\n",
    "        vec_test.append(vec [i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [20/20], Step [17/4], Loss: 0.0003\n",
      "CPU times: user 2min 46s, sys: 39.2 s, total: 3min 25s\n",
      "Wall time: 1min 3s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#тренеруемся\n",
    "\n",
    "hidden_size = 100      \n",
    "num_epochs = 20         \n",
    "batch_size = 400       \n",
    "learning_rate = 0.001  \n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, num_classes):\n",
    "        super(Net, self).__init__()                    \n",
    "        self.fc1 = nn.Linear(input_size, hidden_size)  \n",
    "        self.relu = nn.ReLU()                          \n",
    "        self.fc2 = nn.Linear(hidden_size, num_classes) \n",
    "    \n",
    "    def forward(self, x):                              \n",
    "        out = self.fc1(x)\n",
    "        out = self.relu(out)\n",
    "        out = self.fc2(out)\n",
    "        return out\n",
    "    \n",
    "net = Net(input_size, hidden_size, num_classes)\n",
    "\n",
    "criterion = nn.L1Loss()\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(vec):   #vec_train\n",
    "        images = Variable(images.view(-1, input_size))         \n",
    "        labels = Variable(labels.view(-1, num_classes))\n",
    "\n",
    "        optimizer.zero_grad()                             \n",
    "        outputs = net(images)                             \n",
    "        loss = criterion(outputs, labels)                 \n",
    "        loss.backward()                                   \n",
    "        optimizer.step()                                  \n",
    "        \n",
    "        if (i+1) % 100 == 0:                              \n",
    "            clear_output()\n",
    "            print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f'\n",
    "                 %(epoch+1, num_epochs, (i)//100, len(vec_train)//batch_size, loss.data.item()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "авто/мото:\t\t0.00781187042593956\n",
      "активный отдых:\t\t0.014087893068790436\n",
      "знакомство и общение:\t\t0.6646062135696411\n",
      "игры:\t\t0.024317476898431778\n",
      "ИТ (компьютеры и софт):\t\t0.027786174789071083\n",
      "красота и мода:\t\t0.0009706267155706882\n",
      "литература:\t\t0.6894176006317139\n",
      "мобильная связь и интернет:\t\t0.010921396315097809\n",
      "наука и техника:\t\t0.00718187028542161\n",
      "недвижимость:\t\t0.017370369285345078\n",
      "новости и СМИ:\t\t0.018632175400853157\n",
      "безопасность:\t\t0.008340239524841309\n",
      "обустройство и ремонт:\t\t0.019627729430794716\n",
      "политика:\t\t0.0019035923760384321\n",
      "путешествия:\t\t0.024162745103240013\n",
      "религия:\t\t0.007814777083694935\n",
      "дом и семья:\t\t0.003380067180842161\n",
      "товары и услуги:\t\t0.023995239287614822\n",
      "увлечения и хобби:\t\t0.02548847533762455\n",
      "эзотерика:\t\t0.015736497938632965\n",
      "электроника и бытовая техника:\t\t0.010894142091274261\n"
     ]
    }
   ],
   "source": [
    "testText = 'Подготовка к школе вместе с \"Полиглот\"! В школе начинается английский? Мы подготовим! Приглашаем детей дошкольного возраста и первоклассников на летний интенсив-курс по английскому языку!   Курс рассчитан на обучение английскому детей 6-7 лет и включает в себя азы языка - алфавит, основы чтения и базовую лексику. Благодаря летнему курсу ребенку будет намного легче осваивать новый предмет в школе и адаптироваться к новой социальной среде! Основы грамматики, навыки чтения, усвоенные на уроках, позволят максимально комфортно влиться в школьные занятия. Хотите привить своему ребенку любовь и интерес к новому языку? Приходите, мы точно знаем как это сделать!  Летний интенсив рассчитан на 18 академических часов и займет полтора месяца. Он включает в себя три модуля:'\n",
    "wordsClass.text2Vec(testText)\n",
    "vecOfTheText = torch.zeros(input_size)\n",
    "k = 0\n",
    "for i in range (2000):\n",
    "    if (badIndexes[i]==0):\n",
    "        vecOfTheText[k] = wordsClass.vectorOfStr[i]\n",
    "        k = k + 1\n",
    "\n",
    "data = Variable(vecOfTheText.view(-1, input_size))\n",
    "outputs = net(data)\n",
    "for i in range (42):\n",
    "    if (float (outputs[0][i]) > 0):\n",
    "        print (relationTranslate[i] + ':\\t\\t' + str(float (outputs[0][i])))\n",
    "#print (outputs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}