BERT_FineTuning_Quora_Question_Pairs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Zm2FEueu3skl"
   },
   "source": [
    "# BERT FineTuning on Quora Questions Pairs \n",
    "\n",
    "---\n",
    "\n",
    "In this Colab Notebook, We will try to reproduce state of the art results on Quora Questions Pairs using BERT Model FineTuning. \n",
    "\n",
    "If you are not familiar with BERT, Please visit [The Illustrated BERT](http://jalammar.github.io/illustrated-bert/), [BERT Research Paper](https://arxiv.org/abs/1810.04805) and [BERT Github Repo](https://github.com/google-research/bert).\n",
    "\n",
    "<table class=\"tfo-notebook-buttons\" align=\"left\" >\n",
    " <td>\n",
    "    <a target=\"_blank\" href=\"https://colab.research.google.com/drive/1dCbs4Th3hzJfWEe6KT-stIVDMqHZSA5V\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
    "  </td>\n",
    "  <td>\n",
    "    <a target=\"_blank\" href=\"https://github.com/drc10723/bert_quora_question_pairs\"><img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View Copy on GitHub</a>\n",
    "  </td>\n",
    "</table>\n",
    "\n",
    "\\\n",
    "\n",
    "\\\n",
    "\n",
    "This colab notebook supports both TPU and GPU runtype."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1UmFGb9IHRNh"
   },
   "source": [
    "## Setting Up Environment \n",
    "\n",
    "\n",
    "**USE_TPU :-** True, If you want to use TPU runtime. First change Colab Notebook runtype to TPU\n",
    "\n",
    "**BERT_MODEL:-**  Choose BERT model\n",
    "1.   **uncased_L-12_H-768_A-12**: uncased BERT base model\n",
    "2.   **uncased_L-24_H-1024_A-16**: uncased BERT large model\n",
    "3.   **cased_L-12_H-768_A-12:** cased BERT large model\n",
    "\n",
    "**BUCKET:-** Add bucket details, It is necessary to add bucket for TPU. For GPU runtype, If Bucket is empty, We will use disk.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 170
    },
    "colab_type": "code",
    "id": "u6MTimx2mvKo",
    "outputId": "e3119c4e-ba86-40b6-d991-c6c047603ba5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** BERT pretrained directory: gs://cloud-tpu-checkpoints/bert/uncased_L-12_H-768_A-12 *****\n",
      "gs://cloud-tpu-checkpoints/bert/uncased_L-12_H-768_A-12/bert_config.json\n",
      "gs://cloud-tpu-checkpoints/bert/uncased_L-12_H-768_A-12/bert_model.ckpt.data-00000-of-00001\n",
      "gs://cloud-tpu-checkpoints/bert/uncased_L-12_H-768_A-12/bert_model.ckpt.index\n",
      "gs://cloud-tpu-checkpoints/bert/uncased_L-12_H-768_A-12/bert_model.ckpt.meta\n",
      "gs://cloud-tpu-checkpoints/bert/uncased_L-12_H-768_A-12/checkpoint\n",
      "gs://cloud-tpu-checkpoints/bert/uncased_L-12_H-768_A-12/vocab.txt\n",
      "***** Model output directory: gs://quorabert/outputs *****\n",
      "TPU address is grpc://10.3.4.202:8470\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import json\n",
    "import datetime\n",
    "import pprint\n",
    "import tensorflow as tf\n",
    "\n",
    "# Authenticate, so we can access storage bucket and TPU\n",
    "from google.colab import auth\n",
    "auth.authenticate_user()\n",
    "\n",
    "# If you want to use TPU, first switch to tpu runtime in colab\n",
    "USE_TPU = True #@param{type:\"boolean\"}\n",
    "\n",
    "# We will use base uncased bert model, you can give try with large models\n",
    "# For large model TPU is necessary\n",
    "BERT_MODEL = 'uncased_L-12_H-768_A-12' #@param {type:\"string\"}\n",
    "\n",
    "# BERT checkpoint bucket\n",
    "BERT_PRETRAINED_DIR = 'gs://cloud-tpu-checkpoints/bert/' + BERT_MODEL\n",
    "print('***** BERT pretrained directory: {} *****'.format(BERT_PRETRAINED_DIR))\n",
    "!gsutil ls $BERT_PRETRAINED_DIR\n",
    "\n",
    "# Bucket for saving checkpoints and outputs\n",
    "BUCKET = 'quorabert' #@param {type:\"string\"}\n",
    "if BUCKET!=\"\":\n",
    "  OUTPUT_DIR = 'gs://{}/outputs'.format(BUCKET)\n",
    "  tf.gfile.MakeDirs(OUTPUT_DIR)\n",
    "elif USE_TPU:\n",
    "  raise ValueError('Must specify an existing GCS bucket name for running on TPU')\n",
    "else:\n",
    "  OUTPUT_DIR = 'out_dir'\n",
    "  os.mkdir(OUTPUT_DIR)\n",
    "print('***** Model output directory: {} *****'.format(OUTPUT_DIR))\n",
    "\n",
    "if USE_TPU:\n",
    "  # getting info on TPU runtime\n",
    "  assert 'COLAB_TPU_ADDR' in os.environ, 'ERROR: Not connected to a TPU runtime; Change notebook runtype to TPU'\n",
    "  TPU_ADDRESS = 'grpc://' + os.environ['COLAB_TPU_ADDR']\n",
    "  print('TPU address is', TPU_ADDRESS)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ONIXa1_Pr1xX"
   },
   "source": [
    "## Clone BERT Repo and Download Quora Questions Pairs Dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "id": "0dTKAzm1k5BE",
    "outputId": "b57db3b8-e4c4-434d-c62a-e41048b6b695"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 106400\n",
      "    4 drwxr-xr-x 3 root root     4096 Feb 23 16:28 .\n",
      "    4 drwxr-xr-x 4 root root     4096 Feb 23 16:28 ..\n",
      " 5680 -rw-r--r-- 1 root root  5815716 Feb 23 16:28 dev.tsv\n",
      "    4 drwxr-xr-x 2 root root     4096 Feb 23 16:28 original\n",
      "49572 -rw-r--r-- 1 root root 50759408 Feb 23 16:28 test.tsv\n",
      "51136 -rw-r--r-- 1 root root 52360463 Feb 23 16:28 train.tsv\n"
     ]
    }
   ],
   "source": [
    "# Clone BERT repo and add bert in system path\n",
    "!test -d bert || git clone -q https://github.com/google-research/bert.git\n",
    "if not 'bert' in sys.path:\n",
    "  sys.path += ['bert']\n",
    "# Download QQP Task dataset present in GLUE Tasks.\n",
    "TASK_DATA_DIR = 'glue_data/QQP'\n",
    "!test -d glue_data || git clone https://gist.github.com/60c2bdb54d156a41194446737ce03e2e.git glue_data\n",
    "!test -d $TASK_DATA_DIR || python glue_data/download_glue_data.py --data_dir glue_data --tasks=QQP\n",
    "!ls -als $TASK_DATA_DIR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3nGLW4s-L6ws"
   },
   "source": [
    "## Model Configs and Hyper Parameters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xUNH1_-zHJIH"
   },
   "outputs": [],
   "source": [
    "import modeling\n",
    "import optimization\n",
    "import tokenization\n",
    "import run_classifier\n",
    "\n",
    "# Model Hyper Parameters\n",
    "TRAIN_BATCH_SIZE = 32 # For GPU, reduce to 16\n",
    "EVAL_BATCH_SIZE = 8\n",
    "PREDICT_BATCH_SIZE = 8\n",
    "LEARNING_RATE = 2e-5\n",
    "NUM_TRAIN_EPOCHS = 2.0\n",
    "WARMUP_PROPORTION = 0.1\n",
    "MAX_SEQ_LENGTH = 200 \n",
    "\n",
    "# Model configs\n",
    "SAVE_CHECKPOINTS_STEPS = 1000\n",
    "ITERATIONS_PER_LOOP = 1000\n",
    "NUM_TPU_CORES = 8\n",
    "VOCAB_FILE = os.path.join(BERT_PRETRAINED_DIR, 'vocab.txt')\n",
    "CONFIG_FILE = os.path.join(BERT_PRETRAINED_DIR, 'bert_config.json')\n",
    "INIT_CHECKPOINT = os.path.join(BERT_PRETRAINED_DIR, 'bert_model.ckpt')\n",
    "DO_LOWER_CASE = BERT_MODEL.startswith('uncased')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5RDGdggFQpFB"
   },
   "source": [
    "## Read Questions Pairs\n",
    "\n",
    "We will read data from TSV file and covert to list of InputExample. For `InputExample` and `DataProcessor` class defination refer to [run_classifier](https://github.com/google-research/bert/blob/master/run_classifier.py) file "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5RvBsrOrKLJN"
   },
   "outputs": [],
   "source": [
    "class QQPProcessor(run_classifier.DataProcessor):\n",
    "  \"\"\"Processor for the Quora Question pair data set.\"\"\"\n",
    "\n",
    "  def get_train_examples(self, data_dir):\n",
    "    \"\"\"Reading train.tsv and converting to list of InputExample\"\"\"\n",
    "    return self._create_examples(\n",
    "        self._read_tsv(os.path.join(data_dir,\"train.tsv\")), 'train')\n",
    "\n",
    "  def get_dev_examples(self, data_dir):\n",
    "    \"\"\"Reading dev.tsv and converting to list of InputExample\"\"\"\n",
    "    return self._create_examples(\n",
    "        self._read_tsv(os.path.join(data_dir,\"dev.tsv\")), 'dev')\n",
    "  \n",
    "  def get_test_examples(self, data_dir):\n",
    "    \"\"\"Reading train.tsv and converting to list of InputExample\"\"\"\n",
    "    return self._create_examples(\n",
    "        self._read_tsv(os.path.join(data_dir,\"test.tsv\")), 'test')\n",
    "  \n",
    "  def get_predict_examples(self, sentence_pairs):\n",
    "    \"\"\"Given question pairs, conevrting to list of InputExample\"\"\"\n",
    "    examples = []\n",
    "    for (i, qpair) in enumerate(sentence_pairs):\n",
    "      guid = \"predict-%d\" % (i)\n",
    "      # converting questions to utf-8 and creating InputExamples\n",
    "      text_a = tokenization.convert_to_unicode(qpair[0])\n",
    "      text_b = tokenization.convert_to_unicode(qpair[1])\n",
    "      # We will add label  as 0, because None is not supported in converting to features\n",
    "      examples.append(\n",
    "          run_classifier.InputExample(guid=guid, text_a=text_a, text_b=text_b, label=0))\n",
    "    return examples\n",
    "  \n",
    "  def _create_examples(self, lines, set_type):\n",
    "    \"\"\"Creates examples for the training, dev and test sets.\"\"\"\n",
    "    examples = []\n",
    "    for (i, line) in enumerate(lines):\n",
    "      guid = \"%s-%d\" % (set_type, i)\n",
    "      if set_type=='test':\n",
    "        # removing header and invalid data\n",
    "        if i == 0 or len(line)!=3:\n",
    "          print(guid, line)\n",
    "          continue\n",
    "        text_a = tokenization.convert_to_unicode(line[1])\n",
    "        text_b = tokenization.convert_to_unicode(line[2])\n",
    "        label = 0 # We will use zero for test as convert_example_to_features doesn't support None\n",
    "      else:\n",
    "        # removing header and invalid data\n",
    "        if i == 0 or len(line)!=6:\n",
    "          continue\n",
    "        text_a = tokenization.convert_to_unicode(line[3])\n",
    "        text_b = tokenization.convert_to_unicode(line[4])\n",
    "        label = int(line[5])\n",
    "      examples.append(\n",
    "          run_classifier.InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))\n",
    "    return examples\n",
    "\n",
    "  def get_labels(self):\n",
    "    \"return class labels\"\n",
    "    return [0,1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YRm65m-_ki05"
   },
   "source": [
    "## Convert to Features\n",
    "\n",
    "We will read examples and tokenize using Wordpiece based tokenization. Finally We will convert to `InputFeatures`.\n",
    "\n",
    "BERT follows below tokenization procedure\n",
    "1.   Instantiate an instance of tokenizer = tokenization.FullTokenizer\n",
    "2.   Tokenize the raw text with tokens = tokenizer.tokenize(raw_text).\n",
    "3.   Truncate to the maximum sequence length.\n",
    "4.   Add the [CLS] and [SEP] tokens in the right place.\n",
    "\n",
    "We need to create `segment_ids`, `input_mask` for `InputFeatures`. `segment_ids` will be `0` for question1 tokens and `1` for question2 tokens.\n",
    "\n",
    "We will use following functions from [run_classifier](https://github.com/google-research/bert/blob/master/run_classifier.py) file for converting examples to features :-\n",
    "\n",
    "\n",
    "1.   `convert_single_example` :- Converts a single `InputExample` into a single `InputFeatures`.\n",
    "2.   `file_based_convert_examples_to_features` :- Convert a set of `InputExamples` to a TF_Record file.\n",
    "\n",
    "For more details observe outputs for below cells\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8GMeF2pc7igA"
   },
   "outputs": [],
   "source": [
    "# Instantiate an instance of QQPProcessor and tokenizer\n",
    "processor = QQPProcessor()\n",
    "label_list = processor.get_labels()\n",
    "tokenizer = tokenization.FullTokenizer(vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1278
    },
    "colab_type": "code",
    "id": "OdMc4HkJ7ljr",
    "outputId": "4c2ee1db-0da1-4ca2-dd03-7865563bcb37"
   },
   "outputs": [],
   "source": [
    "# Converting training examples to features\n",
    "print(\"################  Processing Training Data #####################\")\n",
    "TRAIN_TF_RECORD = os.path.join(OUTPUT_DIR, \"train.tf_record\")\n",
    "train_examples = processor.get_train_examples(TASK_DATA_DIR)\n",
    "num_train_examples = len(train_examples)\n",
    "num_train_steps = int( num_train_examples / TRAIN_BATCH_SIZE * NUM_TRAIN_EPOCHS)\n",
    "num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)\n",
    "run_classifier.file_based_convert_examples_to_features(train_examples, label_list, MAX_SEQ_LENGTH, tokenizer, TRAIN_TF_RECORD)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1uq8zO7O5Dnq"
   },
   "source": [
    "## Creating Classification Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6_aUbDJA1N7w"
   },
   "outputs": [],
   "source": [
    "def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,\n",
    "                 labels, num_labels, use_one_hot_embeddings):\n",
    "  \"\"\"Creates a classification model.\"\"\"\n",
    "  # Bert Model instant \n",
    "  model = modeling.BertModel(\n",
    "      config=bert_config,\n",
    "      is_training=is_training,\n",
    "      input_ids=input_ids,\n",
    "      input_mask=input_mask,\n",
    "      token_type_ids=segment_ids,\n",
    "      use_one_hot_embeddings=use_one_hot_embeddings)\n",
    "\n",
    "  # Getting output for last layer of BERT\n",
    "  output_layer = model.get_pooled_output()\n",
    "  \n",
    "  # Number of outputs for last layer\n",
    "  hidden_size = output_layer.shape[-1].value\n",
    "  \n",
    "  # We will use one layer on top of BERT pretrained for creating classification model\n",
    "  output_weights = tf.get_variable(\n",
    "      \"output_weights\", [num_labels, hidden_size],\n",
    "      initializer=tf.truncated_normal_initializer(stddev=0.02))\n",
    "\n",
    "  output_bias = tf.get_variable(\n",
    "      \"output_bias\", [num_labels], initializer=tf.zeros_initializer())\n",
    "\n",
    "  with tf.variable_scope(\"loss\"):\n",
    "    if is_training:\n",
    "      # 0.1 dropout\n",
    "      output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)\n",
    "    \n",
    "    # Calcaulte prediction probabilites and loss\n",
    "    logits = tf.matmul(output_layer, output_weights, transpose_b=True)\n",
    "    logits = tf.nn.bias_add(logits, output_bias)\n",
    "    probabilities = tf.nn.softmax(logits, axis=-1)\n",
    "    log_probs = tf.nn.log_softmax(logits, axis=-1)\n",
    "\n",
    "    one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)\n",
    "\n",
    "    per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)\n",
    "    loss = tf.reduce_mean(per_example_loss)\n",
    "\n",
    "    return (loss, per_example_loss, logits, probabilities)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gxTo8jgbuRoG"
   },
   "source": [
    "## Model Function Builder for Estimator\n",
    "\n",
    "Based on mode, We will create optimizer for training, evaluation metrics for evalution and estimator spec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "An2DFEqX2yDJ"
   },
   "outputs": [],
   "source": [
    "def model_fn_builder(bert_config, num_labels, init_checkpoint, learning_rate,\n",
    "                     num_train_steps, num_warmup_steps, use_tpu,\n",
    "                     use_one_hot_embeddings):\n",
    "  \"\"\"Returns `model_fn` closure for TPUEstimator.\"\"\"\n",
    "\n",
    "  def model_fn(features, labels, mode, params):  \n",
    "    \"\"\"The `model_fn` for TPUEstimator.\"\"\"\n",
    "\n",
    "    # reading features input\n",
    "    input_ids = features[\"input_ids\"]\n",
    "    input_mask = features[\"input_mask\"]\n",
    "    segment_ids = features[\"segment_ids\"]\n",
    "    label_ids = features[\"label_ids\"]\n",
    "    is_real_example = None\n",
    "    if \"is_real_example\" in features:\n",
    "      is_real_example = tf.cast(features[\"is_real_example\"], dtype=tf.float32)\n",
    "    else:\n",
    "      is_real_example = tf.ones(tf.shape(label_ids), dtype=tf.float32)\n",
    "    \n",
    "    # checking if training mode\n",
    "    is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
    "    \n",
    "    # create simple classification model\n",
    "    (total_loss, per_example_loss, logits, probabilities) = create_model(\n",
    "        bert_config, is_training, input_ids, input_mask, segment_ids, label_ids,\n",
    "        num_labels, use_one_hot_embeddings)\n",
    "    \n",
    "    # getting variables for intialization and using pretrained init checkpoint\n",
    "    tvars = tf.trainable_variables()\n",
    "    initialized_variable_names = {}\n",
    "    scaffold_fn = None\n",
    "    if init_checkpoint:\n",
    "      (assignment_map, initialized_variable_names\n",
    "      ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)\n",
    "      if use_tpu:\n",
    "\n",
    "        def tpu_scaffold():\n",
    "          tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n",
    "          return tf.train.Scaffold()\n",
    "\n",
    "        scaffold_fn = tpu_scaffold\n",
    "      else:\n",
    "        tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n",
    "\n",
    "    output_spec = None\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "      # defining optimizar function\n",
    "      train_op = optimization.create_optimizer(\n",
    "          total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu)\n",
    "      \n",
    "      # Training estimator spec\n",
    "      output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n",
    "          mode=mode,\n",
    "          loss=total_loss,\n",
    "          train_op=train_op,\n",
    "          scaffold_fn=scaffold_fn)\n",
    "    elif mode == tf.estimator.ModeKeys.EVAL:\n",
    "      # accuracy, loss, auc, F1, precision and recall metrics for evaluation\n",
    "      def metric_fn(per_example_loss, label_ids, logits, is_real_example):\n",
    "        predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)\n",
    "        loss = tf.metrics.mean(values=per_example_loss, weights=is_real_example)\n",
    "        accuracy = tf.metrics.accuracy(\n",
    "            labels=label_ids, predictions=predictions, weights=is_real_example)\n",
    "        f1_score = tf.contrib.metrics.f1_score(\n",
    "            label_ids,\n",
    "            predictions)\n",
    "        auc = tf.metrics.auc(\n",
    "            label_ids,\n",
    "            predictions)\n",
    "        recall = tf.metrics.recall(\n",
    "            label_ids,\n",
    "            predictions)\n",
    "        precision = tf.metrics.precision(\n",
    "            label_ids,\n",
    "            predictions) \n",
    "        return {\n",
    "            \"eval_accuracy\": accuracy,\n",
    "            \"eval_loss\": loss,\n",
    "            \"f1_score\": f1_score,\n",
    "            \"auc\": auc,\n",
    "            \"precision\": precision,\n",
    "            \"recall\": recall\n",
    "        }\n",
    "\n",
    "      eval_metrics = (metric_fn,\n",
    "                      [per_example_loss, label_ids, logits, is_real_example])\n",
    "      # estimator spec for evalaution\n",
    "      output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n",
    "          mode=mode,\n",
    "          loss=total_loss,\n",
    "          eval_metrics=eval_metrics,\n",
    "          scaffold_fn=scaffold_fn)\n",
    "    else:\n",
    "      # estimator spec for predictions\n",
    "      output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n",
    "          mode=mode,\n",
    "          predictions={\"probabilities\": probabilities},\n",
    "          scaffold_fn=scaffold_fn)\n",
    "    return output_spec\n",
    "\n",
    "  return model_fn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "elGbiKDlamy6"
   },
   "source": [
    "## Creating TPUEstimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "id": "ZPuYpqW97vMf",
    "outputId": "cf38c1db-a1bf-4ea8-e006-2d5a0dc7dbab"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Define TPU configs\n",
    "if USE_TPU:\n",
    "  tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(TPU_ADDRESS)\n",
    "else:\n",
    "  tpu_cluster_resolver = None\n",
    "run_config = tf.contrib.tpu.RunConfig(\n",
    "    cluster=tpu_cluster_resolver,\n",
    "    model_dir=OUTPUT_DIR,\n",
    "    save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS,\n",
    "    tpu_config=tf.contrib.tpu.TPUConfig(\n",
    "        iterations_per_loop=ITERATIONS_PER_LOOP,\n",
    "        num_shards=NUM_TPU_CORES,\n",
    "        per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "f5qq6yfS7yOw"
   },
   "outputs": [],
   "source": [
    "# create model function for estimator using model function builder\n",
    "model_fn = model_fn_builder(\n",
    "    bert_config=modeling.BertConfig.from_json_file(CONFIG_FILE),\n",
    "    num_labels=len(label_list),\n",
    "    init_checkpoint=INIT_CHECKPOINT,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    num_train_steps=num_train_steps,\n",
    "    num_warmup_steps=num_warmup_steps,\n",
    "    use_tpu=USE_TPU,\n",
    "    use_one_hot_embeddings=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 241
    },
    "colab_type": "code",
    "id": "T6D-ZzhlfeGx",
    "outputId": "f6ff8829-b75b-4c04-b4de-dff9c05969ff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f79aa471378>) includes params argument, but params are not passed to Estimator.\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'gs://quorabert/outputs', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "cluster_def {\n",
      "  job {\n",
      "    name: \"worker\"\n",
      "    tasks {\n",
      "      value: \"10.3.4.202:8470\"\n",
      "    }\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f79b558eb00>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': 'grpc://10.3.4.202:8470', '_evaluation_master': 'grpc://10.3.4.202:8470', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver.TPUClusterResolver object at 0x7f79b499e908>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n"
     ]
    }
   ],
   "source": [
    "# Defining TPU Estimator\n",
    "estimator = tf.contrib.tpu.TPUEstimator(\n",
    "    use_tpu=USE_TPU,\n",
    "    model_fn=model_fn,\n",
    "    config=run_config,\n",
    "    train_batch_size=TRAIN_BATCH_SIZE,\n",
    "    eval_batch_size=EVAL_BATCH_SIZE,\n",
    "    predict_batch_size=PREDICT_BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "l1eH8gUIZ9gD"
   },
   "source": [
    "## Finetune Training\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 3641
    },
    "colab_type": "code",
    "id": "Rk4PXAdnjW_N",
    "outputId": "85ee1d1f-1bcd-4a00-a778-e91db6ba2ab5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "QQP on BERT base model normally takes about 1 hour on TPU and 15-20 hours on GPU. Please wait...\n",
      "***** Started training at 2019-02-23 17:50:00.906270 *****\n",
      "  Num examples = 363849\n",
      "  Batch size = 32\n",
      "INFO:tensorflow:  Num steps = 22740\n",
      "INFO:tensorflow:Querying Tensorflow master (grpc://10.3.4.202:8470) for TPU system metadata.\n",
      "INFO:tensorflow:Found TPU system:\n",
      "INFO:tensorflow:*** Num TPU Cores: 8\n",
      "INFO:tensorflow:*** Num TPU Workers: 1\n",
      "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 12018642110560417299)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 5785529947456626056)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 7883019315390049702)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 10377537302524390808)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 16588443086942417458)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 10907570108189753361)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 971213159501917186)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 1464411961604989108)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 1469392857332876480)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 4005027274495629088)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, 12360692828163990549)\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From bert/run_classifier.py:550: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.experimental.map_and_batch(...)`.\n",
      "WARNING:tensorflow:From bert/run_classifier.py:530: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:1720: DatasetV1.make_initializable_iterator (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `for ... in dataset:` to iterate over a dataset. If using `tf.estimator`, return the `Dataset` object directly from your input function. As a last resort, you can use `tf.compat.v1.data.make_initializable_iterator(dataset)`.\n",
      "WARNING:tensorflow:From bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/learning_rate_decay_v2.py:321: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Installing graceful shutdown hook.\n",
      "INFO:tensorflow:Creating heartbeat manager for ['/job:worker/replica:0/task:0/device:CPU:0']\n",
      "INFO:tensorflow:Configuring worker heartbeat: shutdown_mode: WAIT_FOR_COORDINATOR\n",
      "\n",
      "INFO:tensorflow:Init TPU system\n",
      "INFO:tensorflow:Initialized TPU in 4 seconds\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.8386303, step = 1000\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.037307255, step = 2000 (101.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 9.82346\n",
      "INFO:tensorflow:examples/sec: 314.351\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 3000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.023139466, step = 3000 (95.075 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.5179\n",
      "INFO:tensorflow:examples/sec: 336.571\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 4000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.61749035, step = 4000 (97.361 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2711\n",
      "INFO:tensorflow:examples/sec: 328.675\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 5000 into gs://quorabert/outputs/model.ckpt.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to delete files with this prefix.\n",
      "INFO:tensorflow:loss = 1.9537156, step = 5000 (94.422 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.5904\n",
      "INFO:tensorflow:examples/sec: 338.893\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 6000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 1.120376, step = 6000 (95.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.5055\n",
      "INFO:tensorflow:examples/sec: 336.175\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 7000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.84950703, step = 7000 (92.637 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.7948\n",
      "INFO:tensorflow:examples/sec: 345.435\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 8000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0071571404, step = 8000 (95.847 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.4333\n",
      "INFO:tensorflow:examples/sec: 333.865\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 9000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.004391548, step = 9000 (96.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3346\n",
      "INFO:tensorflow:examples/sec: 330.707\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 10000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.011896152, step = 10000 (93.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.7041\n",
      "INFO:tensorflow:examples/sec: 342.531\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 11000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.008425392, step = 11000 (95.291 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.4942\n",
      "INFO:tensorflow:examples/sec: 335.813\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 12000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.8160696, step = 12000 (95.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.5112\n",
      "INFO:tensorflow:examples/sec: 336.357\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 13000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.6142087, step = 13000 (95.948 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.4223\n",
      "INFO:tensorflow:examples/sec: 333.514\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 14000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.4740952, step = 14000 (93.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.6809\n",
      "INFO:tensorflow:examples/sec: 341.789\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 15000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0003787365, step = 15000 (95.261 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.4975\n",
      "INFO:tensorflow:examples/sec: 335.92\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 16000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.010612472, step = 16000 (95.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.5123\n",
      "INFO:tensorflow:examples/sec: 336.394\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 17000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.00039491506, step = 17000 (96.878 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3218\n",
      "INFO:tensorflow:examples/sec: 330.297\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 18000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0002867375, step = 18000 (95.810 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.4377\n",
      "INFO:tensorflow:examples/sec: 334.005\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 19000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0026563092, step = 19000 (98.965 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1047\n",
      "INFO:tensorflow:examples/sec: 323.351\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 20000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0039358316, step = 20000 (93.890 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.6509\n",
      "INFO:tensorflow:examples/sec: 340.829\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 21000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0034660033, step = 21000 (95.893 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.4282\n",
      "INFO:tensorflow:examples/sec: 333.702\n",
      "INFO:tensorflow:Enqueue next (1000) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1000) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Saving checkpoints for 22000 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.9870303, step = 22000 (94.637 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.5667\n",
      "INFO:tensorflow:examples/sec: 338.136\n",
      "INFO:tensorflow:Enqueue next (740) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (740) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:loss = 1.4805896, step = 22740 (55.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.4141\n",
      "INFO:tensorflow:examples/sec: 429.251\n",
      "INFO:tensorflow:Saving checkpoints for 22740 into gs://quorabert/outputs/model.ckpt.\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:Loss for final step: 1.4805896.\n",
      "INFO:tensorflow:training_loop marked as finished\n",
      "***** Finished training at 2019-02-23 18:28:23.504515 *****\n"
     ]
    }
   ],
   "source": [
    "# Train the model.\n",
    "print('QQP on BERT base model normally takes about 1 hour on TPU and 15-20 hours on GPU. Please wait...')\n",
    "print('***** Started training at {} *****'.format(datetime.datetime.now()))\n",
    "print('  Num examples = {}'.format(num_train_examples))\n",
    "print('  Batch size = {}'.format(TRAIN_BATCH_SIZE))\n",
    "tf.logging.info(\"  Num steps = %d\", num_train_steps)\n",
    "# we are using `file_based_input_fn_builder` for creating input function from TF_RECORD file\n",
    "train_input_fn = run_classifier.file_based_input_fn_builder(TRAIN_TF_RECORD,\n",
    "                                                            seq_length=MAX_SEQ_LENGTH,\n",
    "                                                            is_training=True,\n",
    "                                                            drop_remainder=True)\n",
    "estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)\n",
    "print('***** Finished training at {} *****'.format(datetime.datetime.now()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BcSOEcxdZo3B"
   },
   "source": [
    "## Evalute FineTuned model\n",
    "First we will evalute on Train set and Then on Dev set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 887
    },
    "colab_type": "code",
    "id": "ne6yR18I3T09",
    "outputId": "38b1e414-6d46-471f-d2f1-07010ce3fae0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Started Train Set evaluation at 2019-02-23 18:28:23.519756 *****\n",
      "  Num examples = 363849\n",
      "  Batch size = 8\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:363: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-23T18:28:29Z\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from gs://quorabert/outputs/model.ckpt-22740\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Init TPU system\n",
      "INFO:tensorflow:Initialized TPU in 7 seconds\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Enqueue next (45481) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (45481) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Evaluation [45481/45481]\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-23-18:31:52\n",
      "INFO:tensorflow:Saving dict for global step 22740: auc = 0.9406652, eval_accuracy = 0.9413766, eval_loss = 0.254826, f1_score = 0.9219841, global_step = 22740, loss = 0.25182283, precision = 0.90655905, recall = 0.9379432\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 22740: gs://quorabert/outputs/model.ckpt-22740\n",
      "INFO:tensorflow:evaluation_loop marked as finished\n",
      "***** Finished evaluation at 2019-02-23 18:31:55.349592 *****\n",
      "***** Eval results *****\n",
      "  auc = 0.9406652\n",
      "  eval_accuracy = 0.9413766\n",
      "  eval_loss = 0.254826\n",
      "  f1_score = 0.9219841\n",
      "  global_step = 22740\n",
      "  loss = 0.25182283\n",
      "  precision = 0.90655905\n",
      "  recall = 0.9379432\n"
     ]
    }
   ],
   "source": [
    "# eval the model on train set.\n",
    "print('***** Started Train Set evaluation at {} *****'.format(datetime.datetime.now()))\n",
    "print('  Num examples = {}'.format(num_train_examples))\n",
    "print('  Batch size = {}'.format(EVAL_BATCH_SIZE))\n",
    "# eval input function for train set\n",
    "train_eval_input_fn = run_classifier.file_based_input_fn_builder(TRAIN_TF_RECORD,\n",
    "                                                           seq_length=MAX_SEQ_LENGTH,\n",
    "                                                           is_training=False,\n",
    "                                                           drop_remainder=True)\n",
    "# evalute on train set\n",
    "result = estimator.evaluate(input_fn=train_eval_input_fn, \n",
    "                            steps=int(num_train_examples/EVAL_BATCH_SIZE))\n",
    "print('***** Finished evaluation at {} *****'.format(datetime.datetime.now()))\n",
    "print(\"***** Eval results *****\")\n",
    "for key in sorted(result.keys()):\n",
    "  print('  {} = {}'.format(key, str(result[key])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 734
    },
    "colab_type": "code",
    "id": "CAZ_611y7owV",
    "outputId": "e96efaf0-7699-410a-b77c-0c42db0c4c17"
   },
   "outputs": [],
   "source": [
    "# Converting eval examples to features\n",
    "print(\"################  Processing Dev Data #####################\")\n",
    "EVAL_TF_RECORD = os.path.join(OUTPUT_DIR, \"eval.tf_record\")\n",
    "eval_examples = processor.get_dev_examples(TASK_DATA_DIR)\n",
    "num_eval_examples = len(eval_examples)\n",
    "run_classifier.file_based_convert_examples_to_features(eval_examples, label_list, MAX_SEQ_LENGTH, tokenizer, EVAL_TF_RECORD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 785
    },
    "colab_type": "code",
    "id": "rTT5RTAlkCO5",
    "outputId": "bbab37cf-c319-426e-bd21-85a9aa57589a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Started Dev Set evaluation at 2019-02-23 18:32:32.236462 *****\n",
      "  Num examples = 40430\n",
      "  Batch size = 8\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-23T18:32:37Z\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from gs://quorabert/outputs/model.ckpt-22740\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Init TPU system\n",
      "INFO:tensorflow:Initialized TPU in 7 seconds\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Enqueue next (5053) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (5053) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Evaluation [5053/5053]\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-23-18:33:13\n",
      "INFO:tensorflow:Saving dict for global step 22740: auc = 0.89233345, eval_accuracy = 0.896324, eval_loss = 0.4719124, f1_score = 0.86166024, global_step = 22740, loss = 0.47700334, precision = 0.8466528, recall = 0.8772095\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 22740: gs://quorabert/outputs/model.ckpt-22740\n",
      "INFO:tensorflow:evaluation_loop marked as finished\n",
      "***** Finished evaluation at 2019-02-23 18:33:14.659408 *****\n",
      "***** Eval results *****\n",
      "  auc = 0.89233345\n",
      "  eval_accuracy = 0.896324\n",
      "  eval_loss = 0.4719124\n",
      "  f1_score = 0.86166024\n",
      "  global_step = 22740\n",
      "  loss = 0.47700334\n",
      "  precision = 0.8466528\n",
      "  recall = 0.8772095\n"
     ]
    }
   ],
   "source": [
    "# Eval the model on Dev set.\n",
    "print('***** Started Dev Set evaluation at {} *****'.format(datetime.datetime.now()))\n",
    "print('  Num examples = {}'.format(num_eval_examples))\n",
    "print('  Batch size = {}'.format(EVAL_BATCH_SIZE))\n",
    "\n",
    "# eval input function for dev set\n",
    "eval_input_fn = run_classifier.file_based_input_fn_builder(EVAL_TF_RECORD,\n",
    "                                                           seq_length=MAX_SEQ_LENGTH,\n",
    "                                                           is_training=False,\n",
    "                                                           drop_remainder=True)\n",
    "# evalute on dev set\n",
    "result = estimator.evaluate(input_fn=eval_input_fn, steps=int(num_eval_examples/EVAL_BATCH_SIZE))\n",
    "print('***** Finished evaluation at {} *****'.format(datetime.datetime.now()))\n",
    "print(\"***** Eval results *****\")\n",
    "for key in sorted(result.keys()):\n",
    "  print('  {} = {}'.format(key, str(result[key])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zBdomz-EyZKk"
   },
   "source": [
    "\n",
    "## Evaluation Results\n",
    "\n",
    "\n",
    "---\n",
    "Evaluation results are on BERT base uncased model. For reproducing similar results, train for 3 epochs.\n",
    "\n",
    "\n",
    "\n",
    "|**Metrics** | **Train Set** | **Dev Set** |\n",
    "|---|---|---|\n",
    "|**Loss**|0.150|0.497|\n",
    "|**Accuracy**|0.969|0.907|\n",
    "|**F1**|0.959|0.875|\n",
    "|**AUC**|0.969|0.902|\n",
    "|**Precision**|0.949|0.864|\n",
    "|**Recall**|0.969|0.886|\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iOP8xA32CBjE"
   },
   "source": [
    "## Predictions on Model\n",
    "\n",
    "First We will predict on custom examples.\n",
    "\n",
    "For test set, We will get predictions and save in file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AhvA3hEL-2Xt"
   },
   "outputs": [],
   "source": [
    "# examples sentences, feel free to change and try\n",
    "sent_pairs = [(\"how can i improve my english?\", \"how can i become fluent in english?\"), (\"How can i recover old gmail account ?\",\"How can i delete my old gmail account ?\"),\n",
    "             (\"How can i recover old gmail account ?\",\"How can i access my old gmail account ?\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1176
    },
    "colab_type": "code",
    "id": "_CXSUjvgMucd",
    "outputId": "43d926d7-11e1-45f0-ac2e-ae980c628516"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*******  Predictions on Custom Data ********\n",
      "INFO:tensorflow:Writing example 0 of 8\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: predict-0\n",
      "INFO:tensorflow:tokens: [CLS] how can i improve my english ? [SEP] how can i become fluent in english ? [SEP]\n",
      "INFO:tensorflow:input_ids: 101 2129 2064 1045 5335 2026 2394 1029 102 2129 2064 1045 2468 19376 1999 2394 1029 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: predict-1\n",
      "INFO:tensorflow:tokens: [CLS] how can i recover old gma ##il account ? [SEP] how can i del ##ete my old gma ##il account ? [SEP]\n",
      "INFO:tensorflow:input_ids: 101 2129 2064 1045 8980 2214 20917 4014 4070 1029 102 2129 2064 1045 3972 12870 2026 2214 20917 4014 4070 1029 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: predict-2\n",
      "INFO:tensorflow:tokens: [CLS] how can i recover old gma ##il account ? [SEP] how can i access my old gma ##il account ? [SEP]\n",
      "INFO:tensorflow:input_ids: 101 2129 2064 1045 8980 2214 20917 4014 4070 1029 102 2129 2064 1045 3229 2026 2214 20917 4014 4070 1029 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n",
      "  Num examples = 3\n",
      "  Batch size = 8\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from gs://quorabert/outputs/model.ckpt-22740\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Init TPU system\n",
      "INFO:tensorflow:Initialized TPU in 10 seconds\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "[{'probabilities': array([0.9906225, 0.0093775], dtype=float32)}, {'probabilities': array([0.9914073 , 0.00859268], dtype=float32)}, {'probabilities': array([0.004117, 0.995883], dtype=float32)}, {'probabilities': array([0.00136185, 0.9986381 ], dtype=float32)}, {'probabilities': array([0.00136185, 0.9986381 ], dtype=float32)}, {'probabilities': array([0.00136185, 0.9986381 ], dtype=float32)}, {'probabilities': array([0.00136185, 0.9986381 ], dtype=float32)}, {'probabilities': array([0.00136185, 0.9986381 ], dtype=float32)}]\n",
      "****** Example 0 ******\n",
      "Question1 : how can i improve my english?\n",
      "Question2 : how can i become fluent in english?\n",
      "Prediction : 0.009377497\n",
      "****** Example 1 ******\n",
      "Question1 : How can i recover old gmail account ?\n",
      "Question2 : How can i delete my old gmail account ?\n",
      "Prediction : 0.008592678\n",
      "****** Example 2 ******\n",
      "Question1 : How can i recover old gmail account ?\n",
      "Question2 : How can i access my old gmail account ?\n",
      "Prediction : 0.995883\n"
     ]
    }
   ],
   "source": [
    "print(\"*******  Predictions on Custom Data ********\")\n",
    "# create `InputExample` for custom examples\n",
    "predict_examples = processor.get_predict_examples(sent_pairs)\n",
    "num_predict_examples = len(predict_examples)\n",
    "\n",
    "# For TPU, We will append `PaddingExample` for maintaining batch size\n",
    "if USE_TPU:\n",
    "  while(len(predict_examples)%EVAL_BATCH_SIZE!=0):\n",
    "    predict_examples.append(run_classifier.PaddingInputExample())\n",
    "\n",
    "# Converting to features \n",
    "predict_features = run_classifier.convert_examples_to_features(predict_examples, label_list, MAX_SEQ_LENGTH, tokenizer)\n",
    "\n",
    "print('  Num examples = {}'.format(num_predict_examples))\n",
    "print('  Batch size = {}'.format(PREDICT_BATCH_SIZE))\n",
    "\n",
    "# Input function for prediction\n",
    "predict_input_fn = run_classifier.input_fn_builder(predict_features,\n",
    "                                                seq_length=MAX_SEQ_LENGTH,\n",
    "                                                is_training=False,\n",
    "                                                drop_remainder=True)\n",
    "result = list(estimator.predict(input_fn=predict_input_fn))\n",
    "print(result)\n",
    "for ex_i in range(num_predict_examples):\n",
    "  print(\"****** Example {} ******\".format(ex_i))\n",
    "  print(\"Question1 :\", sent_pairs[ex_i][0])\n",
    "  print(\"Question2 :\", sent_pairs[ex_i][1])\n",
    "  print(\"Prediction :\", result[ex_i]['probabilities'][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 751
    },
    "colab_type": "code",
    "id": "6TOq8o9aTMzk",
    "outputId": "312bd5c4-aad1-4fa5-8d39-3aa00770b883"
   },
   "outputs": [],
   "source": [
    "# Converting test examples to features\n",
    "print(\"################  Processing Test Data #####################\")\n",
    "TEST_TF_RECORD = os.path.join(OUTPUT_DIR, \"test.tf_record\")\n",
    "test_examples = processor.get_test_examples(TASK_DATA_DIR)\n",
    "num_test_examples = len(test_examples)\n",
    "run_classifier.file_based_convert_examples_to_features(test_examples, label_list, MAX_SEQ_LENGTH, tokenizer, TEST_TF_RECORD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-5ohC_ab0i8I"
   },
   "outputs": [],
   "source": [
    "# Predictions on test set.\n",
    "print('***** Started Prediction at {} *****'.format(datetime.datetime.now()))\n",
    "print('  Num examples = {}'.format(num_test_examples))\n",
    "print('  Batch size = {}'.format(PREDICT_BATCH_SIZE))\n",
    "# predict input function for test set\n",
    "test_input_fn = run_classifier.file_based_input_fn_builder(TEST_TF_RECORD,\n",
    "                                                           seq_length=MAX_SEQ_LENGTH,\n",
    "                                                           is_training=False,\n",
    "                                                           drop_remainder=True)\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "# predict on test set\n",
    "result = list(estimator.predict(input_fn=test_input_fn))\n",
    "print('***** Finished Prediction at {} *****'.format(datetime.datetime.now()))\n",
    "\n",
    "# saving test predictions\n",
    "output_test_file = os.path.join(OUTPUT_DIR, \"test_predictions.txt\")\n",
    "with tf.gfile.GFile(output_test_file, \"w\") as writer:\n",
    "  for (example_i, predictions_i) in enumerate(result):\n",
    "    writer.write(\"%s , %s\\n\" % (test_examples[example_i].guid, str(predictions_i['probabilities'][1])))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "collapsed_sections": [],
   "name": "BERT FineTuning Quora Question Pairs.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}