code_session_version/lab-12-4-many-to-many-bidirectional-session.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-4 many to many variable bidirectional\n",
    "### simple pos-tagger training \n",
    "* many to many\n",
    "* variable input sequence length\n",
    "* bi-directional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.nn import embedding_lookup\n",
    "from tensorflow.nn.rnn_cell import *\n",
    "from tensorflow.nn import bidirectional_dynamic_rnn\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "pos = [['pronoun', 'verb', 'adjective'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective'],\n",
    "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
      "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# word의 dictionary\n",
    "word_list = sum(sentences, [])\n",
    "word_list = sorted(set(word_list))\n",
    "word_list = ['<pad>'] + word_list\n",
    "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
    "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
    "\n",
    "print(word2idx)\n",
    "print(idx2word)\n",
    "print(len(idx2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
      "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# pos (part of speech)의 dictionary\n",
    "pos_list = sum(pos, [])\n",
    "pos_list = sorted(set(pos_list))\n",
    "pos_list = ['<pad>'] + pos_list\n",
    "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
    "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
    "\n",
    "print(pos2idx)\n",
    "print(idx2pos)\n",
    "print(len(pos2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_sequence = 10\n",
    "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence,padding='post')\n",
    "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
    "x_data_len = np.array(list(map(lambda sentence : len(sentence), sentences)))\n",
    "\n",
    "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
    "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence,padding='post')\n",
    "# y_data = np.expand_dims(y_data, -1)\n",
    "# y_data = np.apply_along_axis(lambda row : to_categorical(row), -1, y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, x, x_len, y, hidden_dim, num_classes, max_sequence, dic):\n",
    "        # input\n",
    "        self.x = x\n",
    "        self.x_len = x_len\n",
    "        self.y = y\n",
    "        self.dic = dic\n",
    "        \n",
    "        # token representation\n",
    "        one_hot = tf.Variable(np.eye(len(self.dic)), dtype=tf.float32, trainable=False)\n",
    "        x_batch = embedding_lookup(params=one_hot, ids=self.x)        \n",
    "        \n",
    "        # rnn\n",
    "        fw_rnn_cell = BasicRNNCell(num_units=hidden_dim, activation=tf.nn.tanh)\n",
    "        bw_rnn_cell = BasicRNNCell(num_units=hidden_dim, activation=tf.nn.tanh)\n",
    "        outputs, _= bidirectional_dynamic_rnn(cell_fw=fw_rnn_cell, cell_bw=bw_rnn_cell,\n",
    "                                              inputs=x_batch, sequence_length=self.x_len,\n",
    "                                              dtype=tf.float32)\n",
    "        concatenated_outputs = tf.concat([outputs[0], outputs[1]], axis = -1)\n",
    "        \n",
    "        # classifier\n",
    "        weights = tf.Variable(initial_value=tf.random_normal(shape=[2 * hidden_dim, num_classes]),\n",
    "                              name='weights', dtype=tf.float32)\n",
    "        self.score = tf.map_fn(lambda elm : tf.matmul(elm, weights), concatenated_outputs)\n",
    "        \n",
    "        # loss\n",
    "        masking = tf.sequence_mask(lengths=self.x_len, maxlen=max_sequence, dtype=tf.float32)\n",
    "        self.sequence_loss = tf.contrib.seq2seq.sequence_loss(logits=self.score, targets=y,\n",
    "                                                              weights=masking)\n",
    "        \n",
    "        # prediction\n",
    "        self.prediction = tf.argmax(input=self.score, axis=-1)\n",
    "        \n",
    "    def predict(self, sess, x, x_len):\n",
    "        feed_prediction = {self.x : x, self.x_len : x_len}\n",
    "        return sess.run(self.prediction, feed_dict=feed_prediction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((?, 10), (?, 10), (?,)), types: (tf.int32, tf.int32, tf.int64)>\n",
      "WARNING:tensorflow:From <ipython-input-6-aa571148f5a0>:14: BasicRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.SimpleRNNCell, and will be replaced by that in Tensorflow 2.0.\n"
     ]
    }
   ],
   "source": [
    "# Creating a training operation\n",
    "lr = .1\n",
    "epochs = 30\n",
    "batch_size = 2\n",
    "\n",
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data,y_data,x_data_len))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size = 4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = 2)\n",
    "\n",
    "print(tr_dataset)\n",
    "tr_iterator = tr_dataset.make_initializable_iterator()\n",
    "x_mb, y_mb, x_mb_len = tr_iterator.get_next()\n",
    "\n",
    "# creating model\n",
    "model = Model(x=x_mb, x_len=x_mb_len, y=y_mb, hidden_dim=10, num_classes=len(pos2idx),\n",
    "              max_sequence=10, dic=word2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = tf.train.AdamOptimizer(learning_rate = lr)\n",
    "training_op = opt.minimize(loss=model.sequence_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.012\n",
      "epoch :  10, tr_loss : 0.002\n",
      "epoch :  15, tr_loss : 0.000\n",
      "epoch :  20, tr_loss : 0.000\n",
      "epoch :  25, tr_loss : 0.000\n",
      "epoch :  30, tr_loss : 0.000\n"
     ]
    }
   ],
   "source": [
    "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))\n",
    "sess = tf.Session(config = sess_config)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    # training\n",
    "    sess.run(tr_iterator.initializer)\n",
    "    try:\n",
    "        while True:\n",
    "            _, tr_loss = sess.run([training_op, model.sequence_loss])\n",
    "            avg_tr_loss += tr_loss\n",
    "            tr_step += 1\n",
    "    except tf.errors.OutOfRangeError:\n",
    "        avg_tr_loss /= tr_step\n",
    "        \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch+1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = model.predict(sess, x_data, x_data_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6, 7, 1, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [4, 7, 2, 1, 0, 0, 0, 0, 0, 0],\n",
       "       [4, 7, 3, 4, 5, 1, 4, 0, 0, 0],\n",
       "       [4, 7, 2, 1, 7, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6, 7, 1, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [4, 7, 2, 1, 0, 0, 0, 0, 0, 0],\n",
       "       [4, 7, 3, 4, 5, 1, 4, 0, 0, 0],\n",
       "       [4, 7, 2, 1, 7, 0, 0, 0, 0, 0]], dtype=int32)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}