keras_lstm_words.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from keras.preprocessing import sequence\n",
    "from keras.optimizers import SGD, RMSprop, Adagrad\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation, Embedding, RepeatVector, SharedDense\n",
    "from keras.activations import time_distributed_softmax\n",
    "from keras.layers.recurrent import LSTM, GRU\n",
    "import theano\n",
    "theano.config.optimizer='fast_compile'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "filename = \"/media/charles/data/articles\"\n",
    "h = open(filename)\n",
    "all_jsons=[]\n",
    "for line in h:    \n",
    "    if line[0]=='[':\n",
    "        all_jsons.append(line[:-1])\n",
    "        \n",
    "TitleList = []\n",
    "#TextList = []\n",
    "IndexList = []\n",
    "count = 0\n",
    "\n",
    "for oneJson in all_jsons:\n",
    "    u = json.loads(oneJson)\n",
    "    for item in u:\n",
    "        fields = item['fields']\n",
    "        TitleList.append(fields['title'].encode('ascii','ignore'))\n",
    "        #TextList.append(fields['text'].encode('ascii','ignore'))\n",
    "        IndexList.append(item['rowKey'].encode('ascii','ignore'))\n",
    "        count+=1\n",
    "        if count%10000==0:\n",
    "            print(\"done: \"+str(count))\n",
    "\n",
    "all_jsons = []\n",
    "del all_jsons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import np_utils\n",
    "\n",
    "max_words = 10000\n",
    "batch_size = 16\n",
    "\n",
    "print \"Loading data...\"\n",
    "X_seq = TitleList[0:60000]\n",
    "\n",
    "print len(X_seq), 'total sequences'\n",
    "\n",
    "print \"Vectorizing sequence data...\"\n",
    "tokenizer = Tokenizer(nb_words=max_words)\n",
    "tokenizer.fit_on_texts(X_seq, unknown_words=True)\n",
    "X_seq = tokenizer.texts_to_sequences(X_seq, replace=True)\n",
    "\n",
    "print \"Divide train and test\"\n",
    "y_train = []\n",
    "x_train = []\n",
    "maxlen = 50\n",
    "\n",
    "for s in X_seq:\n",
    "    if len(s) < 3 or len(s)>maxlen:\n",
    "        continue        \n",
    "    x_train.append(s)\n",
    "    y_train.append(s[::-1])\n",
    "\n",
    "\n",
    "#Y_train = np_utils.to_categorical(y_train, max_words)\n",
    "    \n",
    "cut = int(len(X_seq)*0.8)\n",
    "X_test = x_train[cut:]\n",
    "X_train = x_train[:cut]\n",
    "Y_test = y_train[cut:]\n",
    "Y_train = y_train[:cut]\n",
    "    \n",
    "print \"Pad sequences (samples x time)\"\n",
    "X_train = sequence.pad_sequences(X_train, maxlen=maxlen)\n",
    "X_test = sequence.pad_sequences(X_test, maxlen=maxlen)\n",
    "Y_train = sequence.pad_sequences(Y_train, maxlen=maxlen)\n",
    "Y_test = sequence.pad_sequences(Y_test, maxlen=maxlen)\n",
    "\n",
    "print 'X_train shape:', X_train.shape\n",
    "print 'X_test shape:', X_test.shape\n",
    "\n",
    "#print \"Convert class vector to binary class matrix (for use with categorical_crossentropy)\"\n",
    "#Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
    "#Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
    "#print 'Y_train shape:', Y_train.shape\n",
    "#print 'Y_test shape:', Y_test.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "voc_dic = tokenizer.word_index.items()\n",
    "voc_dic.sort(key = lambda x: x[1])\n",
    "voc = [x[0] for x in voc_dic]\n",
    "inv_dic = {v:k for (k,v) in voc_dic}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "voc = np.asarray(voc)\n",
    "np.savez_compressed('dataset',x_train=X_train,x_test=X_test,y_train=Y_train,y_test=Y_test,voc=voc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['x_test', 'x_train', 'y_train', 'y_test', 'voc']\n",
      "10000\n"
     ]
    }
   ],
   "source": [
    "x = np.load('dataset.npz')\n",
    "print x.files\n",
    "voc = x['voc'].tolist()\n",
    "voc_dic = dict(zip(voc, range(len(voc))))\n",
    "inv_dic = {v:k for (k,v) in voc_dic.items()}\n",
    "X_train, X_test, Y_train, Y_test = x['x_train'],x['x_test'],x['y_train'],x['y_test']\n",
    "max_words = max(np.max(X_train), np.max(X_test))+1\n",
    "print max_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n"
     ]
    }
   ],
   "source": [
    "max_features= max_words\n",
    "emb_size = 256\n",
    "lstm_size = 128\n",
    "batch_size = 16\n",
    "max_size = X_train.shape[1]\n",
    "\n",
    "print 'Build model...'\n",
    "model = Sequential()\n",
    "model.add(Embedding(max_features, emb_size))\n",
    "model.add(LSTM(emb_size, lstm_size)) # try using a GRU instead, for fun\n",
    "model.add(RepeatVector(max_size))\n",
    "#model.add(Dropout(0.5))\n",
    "model.add(LSTM(lstm_size, lstm_size, return_sequences=True))\n",
    "#model.add(Activation('time_distributed_softmax'))\n",
    "model.add(SharedDense(lstm_size, max_features, activation='softmax'))\n",
    "optimizer = SGD(lr=0.7/batch_size)\n",
    "\n",
    "\n",
    "model.compile(loss='categorical_scalar_crossentropy', optimizer=optimizer)\n",
    "# try using different optimizers and different optimizer configs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train...\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-8724b81dbb3a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mprint\u001b[0m \u001b[1;34m\"Train...\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnb_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mscore\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mprint\u001b[0m \u001b[1;34m'Test score:'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscore\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/charles/Programs/keras/keras/models.pyc\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, batch_size, nb_epoch, verbose, validation_split, shuffle)\u001b[0m\n\u001b[0;32m     89\u001b[0m                 \u001b[0mX_batch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     90\u001b[0m                 \u001b[0my_batch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 91\u001b[1;33m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_train\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_batch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     92\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     93\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m>\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/Theano-0.6.0-py2.7.egg/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    577\u001b[0m         \u001b[0mt0_fn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    578\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 579\u001b[1;33m             \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    580\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    581\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'position_of_error'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/Theano-0.6.0-py2.7.egg/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mrval\u001b[1;34m(p, i, o, n)\u001b[0m\n\u001b[0;32m    649\u001b[0m         \u001b[1;31m# default arguments are stored in the closure of `rval`\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    650\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 651\u001b[1;33m         \u001b[1;32mdef\u001b[0m \u001b[0mrval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnode_input_storage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnode_output_storage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    652\u001b[0m             \u001b[0mr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    653\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "print \"Train...\"\n",
    "model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=10, verbose=1)\n",
    "score = model.evaluate(X_test, Y_test, batch_size=batch_size)\n",
    "print 'Test score:', score\n",
    "\n",
    "classes = model.predict_classes(X_test, batch_size=batch_size)\n",
    "acc = np_utils.accuracy(classes, Y_test)\n",
    "print 'Test accuracy:', acc\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tostr(u):\n",
    "    st = \"\"\n",
    "    for x in u:\n",
    "        st+=inv_dic[x]+\" \"\n",
    "    return st.strip()\n",
    "\n",
    "def see(X, Y):\n",
    "    for x,y in zip (X,Y):\n",
    "        print tostr(x), \" / \", tostr(y)\n",
    "\n",
    "def seeModel(X, model):\n",
    "    see(X, model.predict_classes(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "seeModel(X_test[0:20], model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#x = model._predict(X_test[0:16])\n",
    "x = model.predict_proba(X_test[0:16], batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model.predict_classes(X_test[0:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tostr(X_test[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "(16,) + x[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}