01-Phonemes-to-words.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demonstrate Seq2Seq Wrapper with CMUDict dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "# preprocessed data\n",
    "from datasets.cmudict import data\n",
    "import data_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load data from pickle and npy files\n",
    "data_ctl, idx_words, idx_phonemes = data.load_data(PATH='datasets/cmudict/')\n",
    "(trainX, trainY), (testX, testY), (validX, validY) = data_utils.split_dataset(idx_phonemes, idx_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# parameters \n",
    "xseq_len = trainX.shape[-1]\n",
    "yseq_len = trainY.shape[-1]\n",
    "batch_size = 128\n",
    "xvocab_size = len(data_ctl['idx2pho'].keys())  \n",
    "yvocab_size = len(data_ctl['idx2alpha'].keys())\n",
    "emb_dim = 128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create an instance of the Wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import seq2seq_wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'seq2seq_wrapper' from '/home/suriya/_/tf/tf-seq2seq-wrapper/seq2seq_wrapper.py'>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import importlib\n",
    "importlib.reload(seq2seq_wrapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<log> Building Graph </log>"
     ]
    }
   ],
   "source": [
    "model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,\n",
    "                               yseq_len=yseq_len,\n",
    "                               xvocab_size=xvocab_size,\n",
    "                               yvocab_size=yvocab_size,\n",
    "                               ckpt_path='ckpt/cmudict/',\n",
    "                               emb_dim=emb_dim,\n",
    "                               num_layers=3\n",
    "                               )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create data generators\n",
    "\n",
    "Read *data_utils.py* for more information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_batch_gen = data_utils.rand_batch_gen(validX, validY, 16)\n",
    "train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, 128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Computational graph was built when the model was instantiated\n",
    "- Now all we need to do is train the model using processed CMUdict dataset, via data generators\n",
    "- Internally a loop is run for *epochs* times for training\n",
    "- Evaluation is done periodically."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model saved to disk at iteration #5000\n",
      "val   loss : 0.428838\n",
      "\n",
      "Model saved to disk at iteration #10000\n",
      "val   loss : 0.352279\n",
      "\n",
      "Model saved to disk at iteration #15000\n",
      "val   loss : 0.302959\n",
      "\n",
      "Model saved to disk at iteration #20000\n",
      "val   loss : 0.290396\n",
      "\n",
      "Model saved to disk at iteration #25000\n",
      "val   loss : 0.250649\n",
      "\n",
      "Model saved to disk at iteration #30000\n",
      "val   loss : 0.239168\n",
      "\n",
      "Model saved to disk at iteration #35000\n",
      "val   loss : 0.198182\n",
      "\n",
      "Model saved to disk at iteration #40000\n",
      "val   loss : 0.203086\n",
      "\n",
      "Model saved to disk at iteration #45000\n",
      "val   loss : 0.213277\n",
      "\n",
      "Model saved to disk at iteration #50000\n",
      "val   loss : 0.208600\n",
      "\n",
      "Model saved to disk at iteration #55000\n",
      "val   loss : 0.228991\n",
      "\n",
      "Model saved to disk at iteration #60000\n",
      "val   loss : 0.205643\n",
      "Interrupted by user at iteration 60001\n"
     ]
    }
   ],
   "source": [
    "sess = model.train(train_batch_gen, val_batch_gen, sess=sess1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Restore last saved session from disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sess = model.restore_last_session()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(16, 16)\n"
     ]
    }
   ],
   "source": [
    "output = model.predict(sess, val_batch_gen.__next__()[0])\n",
    "print(output.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[12, 15, 14,  7, 23,  5, 12, 12,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 8, 15, 12, 19, 23, 15, 18, 20,  8,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 3,  1, 22,  1, 12,  9,  5, 18,  9,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [16, 18, 15, 19,  5, 12,  9, 20,  9, 26,  5,  0,  0,  0,  0,  0],\n",
       "       [ 8,  9,  7,  1, 19,  8,  9,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 4,  9, 19,  2,  9, 12,  5,  9,  6,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [11,  9, 19, 13,  5, 20,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [21, 14,  3, 15, 20,  5,  4,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [19, 21,  3,  8,  9,  1, 11,  9,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [13,  5, 20, 18, 15,  3, 15, 12,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 4, 21, 18, 21, 19,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [13,  1,  7, 14, 15, 12,  9,  1,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [12,  5, 20, 20,  1, 18, 20,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 7,  1, 12, 12,  1, 23,  1, 25,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 2, 18,  9,  1, 14,  5,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 8, 15, 18, 16,  9,  6,  9, 14,  9,  0,  0,  0,  0,  0,  0,  0]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let us decode and see the words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "longwell\n",
      "holsworth\n",
      "cavalieri\n",
      "proselitize\n",
      "higashi\n",
      "disbileif\n",
      "kismet\n",
      "uncoted\n",
      "suchiaki\n",
      "metrocol\n",
      "durus\n",
      "magnolia\n",
      "lettart\n",
      "gallaway\n",
      "briane\n",
      "horpifini\n"
     ]
    }
   ],
   "source": [
    "for oi in output:\n",
    "    print(data_utils.decode(sequence=oi, lookup=data_ctl['idx2alpha'],\n",
    "                           separator=''))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}