BERT_base_imdb_hugging.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT_base_imdb_hugging.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_FXneEfpdMM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 347
        },
        "outputId": "79fd4814-1cb6-48eb-a006-2be42f555d85"
      },
      "source": [
        "!pip install pytorch_pretrained_bert pytorch-nlp"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pytorch_pretrained_bert in /usr/local/lib/python3.6/dist-packages (0.6.2)\n",
            "Requirement already satisfied: pytorch-nlp in /usr/local/lib/python3.6/dist-packages (0.5.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (2019.12.20)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.14.24)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.18.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (2.23.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (4.41.1)\n",
            "Requirement already satisfied: torch>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.5.1+cu101)\n",
            "Requirement already satisfied: botocore<1.18.0,>=1.17.24 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (1.17.24)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (0.10.0)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (0.3.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (1.24.3)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=0.4.1->pytorch_pretrained_bert) (0.16.0)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.24->boto3->pytorch_pretrained_bert) (2.8.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.24->boto3->pytorch_pretrained_bert) (0.15.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.18.0,>=1.17.24->boto3->pytorch_pretrained_bert) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kew08PWs4TAS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 347
        },
        "outputId": "4b3affcd-9ede-43d6-eac8-eeed66e9485e"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.0.2)\n",
            "Requirement already satisfied: tokenizers==0.8.1.rc1 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8.1rc1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "EnVIV6Vt8f4d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1a7bdb43-5815-422b-8eeb-c8b0fb7e291e"
      },
      "source": [
        "import sys\n",
        "import numpy as np\n",
        "import random as rn\n",
        "import torch\n",
        "from torch import nn\n",
        "\n",
        "from transformers import BertModel\n",
        "from transformers import BertTokenizer\n",
        "from transformers import BertConfig\n",
        "\n",
        "\n",
        "from torchnlp.datasets import imdb_dataset\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from torch.optim import Adam\n",
        "from torch.nn.utils import clip_grad_norm_\n",
        "from IPython.display import clear_output"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cUYrv06z8gaF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "rn.seed(321)\n",
        "np.random.seed(321)\n",
        "torch.manual_seed(321)\n",
        "torch.cuda.manual_seed(321)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rgkbhHcB17GY",
        "colab_type": "text"
      },
      "source": [
        "## Prepare the Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ur8i7boP6qtb",
        "colab": {}
      },
      "source": [
        "train_data, test_data = imdb_dataset(train=True, test=True)\n",
        "rn.shuffle(train_data)\n",
        "rn.shuffle(test_data)\n",
        "train_data = train_data[:1000]\n",
        "test_data = test_data[:100]"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l1VENo6tqG7J",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b33afc62-813f-40d0-f49e-932c38b71c25"
      },
      "source": [
        "train_texts, train_labels = list(zip(*map(lambda d: (d['text'], d['sentiment']), train_data)))\n",
        "test_texts, test_labels = list(zip(*map(lambda d: (d['text'], d['sentiment']), test_data)))\n",
        "\n",
        "len(train_texts), len(train_labels), len(test_texts), len(test_labels)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 1000, 100, 100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ty24UrRjqIsb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "26trq3gIrJeG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4e16ca23-256a-4d9e-ab71-f226c4b8a4ca"
      },
      "source": [
        "tokenizer.tokenize('Hi my name is Dima')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['hi', 'my', 'name', 'is', 'dim', '##a']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1k9rcOzQr5Zm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#train_tokens = list(map(lambda t: ['[CLS]'] + tokenizer.tokenize(t)[:510] + ['[SEP]'], train_texts))\n",
        "#test_tokens = list(map(lambda t: ['[CLS]'] + tokenizer.tokenize(t)[:510] + ['[SEP]'], test_texts))\n",
        "\n",
        "#len(train_tokens), len(test_tokens)                   \n",
        "                   "
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Ca7KKnhuT5c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#train_tokens_ids = pad_sequences(list(map(tokenizer.convert_tokens_to_ids, train_tokens)), maxlen=512, truncating=\"post\", padding=\"post\", dtype=\"int\")\n",
        "#test_tokens_ids = pad_sequences(list(map(tokenizer.convert_tokens_to_ids, test_tokens)), maxlen=512, truncating=\"post\", padding=\"post\", dtype=\"int\")\n",
        "\n",
        "#train_tokens_ids.shape, test_tokens_ids.shape"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-1tQ-uovfllt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_train = tokenizer(train_texts, padding=True, truncation=True, return_tensors=\"pt\")\n",
        "batch_test = tokenizer(test_texts, padding=True, truncation=True, return_tensors=\"pt\")"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AKBkG-Z5fosF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_tokens_ids = batch_train['input_ids']\n",
        "test_tokens_ids = batch_test['input_ids']"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fv3W8VCAgFNS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_masks = batch_train['attention_mask']\n",
        "test_masks = batch_test['attention_mask']"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jsobdjhxieqz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZlPjalrNjCz2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aL4NlQQsfo15",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "F7POtHuIOV-6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e91bea45-b16c-428a-debb-7d3463f75691"
      },
      "source": [
        "train_y = np.array(train_labels) == 'pos'\n",
        "test_y = np.array(test_labels) == 'pos'\n",
        "train_y.shape, test_y.shape, np.mean(train_y), np.mean(test_y)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((1000,), (100,), 0.489, 0.5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U-xXMEqXOWTE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#train_masks = [[float(i > 0) for i in ii] for ii in train_tokens_ids]\n",
        "#test_masks = [[float(i > 0) for i in ii] for ii in test_tokens_ids]"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2K4JQMFo1-_S",
        "colab_type": "text"
      },
      "source": [
        "# Baseline"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wdzjl_WlwpKr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.metrics import classification_report"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9jyb-hJ0xAgG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "outputId": "de75d5fd-9a34-4b22-e280-5df5234ff807"
      },
      "source": [
        "baseline_model = make_pipeline(CountVectorizer(ngram_range=(1,3)), LogisticRegression()).fit(train_texts, train_labels)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q9IzjAX_2VLf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "baseline_predicted = baseline_model.predict(test_texts)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QnsCRIaQ3GPQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "cb47dc96-a2d7-40a7-d0f8-5dd3b20a2eca"
      },
      "source": [
        "print(classification_report(test_labels, baseline_predicted))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "         neg       0.80      0.82      0.81        50\n",
            "         pos       0.82      0.80      0.81        50\n",
            "\n",
            "    accuracy                           0.81       100\n",
            "   macro avg       0.81      0.81      0.81       100\n",
            "weighted avg       0.81      0.81      0.81       100\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r_hEhebQ3YqI",
        "colab_type": "text"
      },
      "source": [
        "# Bert Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E234ByBa3Qtb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BertBinaryClassifier(nn.Module):\n",
        "    def __init__(self, dropout=0.1):\n",
        "        super(BertBinaryClassifier, self).__init__()\n",
        "\n",
        "        #model_config = BertConfig.from_pretrained('bert-base-uncased', output_hidden_states=True)\n",
        "        self.bert = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        self.linear = nn.Linear(768, 1)\n",
        "        self.sigmoid = nn.Sigmoid()\n",
        "    \n",
        "    def forward(self, tokens, masks=None):\n",
        "        _, pooled_output = self.bert(tokens, attention_mask=masks)\n",
        "        dropout_output = self.dropout(pooled_output)\n",
        "        linear_output = self.linear(dropout_output)\n",
        "        proba = self.sigmoid(linear_output)\n",
        "        return proba\n",
        "        "
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ED9SE1Ka8W9x",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "568ac52a-d5c2-4745-e4af-dddf845b8711"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "device"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FNZ_3auqDbjl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "03513727-93dd-474e-e66d-6e1d1f57132b"
      },
      "source": [
        "str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'0.0M'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Sf9n8zouENRi",
        "colab": {}
      },
      "source": [
        "bert_clf = BertBinaryClassifier()\n",
        "bert_clf = bert_clf.cuda()\n"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LbHkaJuZEkYr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "3b114836-4384-4b7d-8056-e00354dee71c"
      },
      "source": [
        "str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'439.065088M'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LOQ-870M7VWy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "outputId": "7dff3084-a7bf-457d-cdc4-2d28ff54196f"
      },
      "source": [
        "x = torch.tensor(train_tokens_ids[:3]).to(device)\n",
        "y, pooled = bert_clf.bert(x)\n",
        "x.shape, y.shape, pooled.shape"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([3, 512]), torch.Size([3, 512, 768]), torch.Size([3, 768]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LCb_pK4X7hb9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "7371042f-ec61-4ec3-c7b1-fc11727289bb"
      },
      "source": [
        "y = bert_clf(x)\n",
        "y.cpu().detach().numpy()"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.6604583 ],\n",
              "       [0.6673066 ],\n",
              "       [0.59659123]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MUm-gFCuFkoI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "6fefd70f-6085-467e-97b7-5fc2ece18cba"
      },
      "source": [
        "str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'4082.508288M'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KzzsUZOUFcxp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "f2ba2d25-4b1b-4fab-9f86-c64efd74d2b3"
      },
      "source": [
        "y, x, pooled = None, None, None\n",
        "torch.cuda.empty_cache()\n",
        "str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'439.065088M'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c9LPIYcn99r8",
        "colab_type": "text"
      },
      "source": [
        "# Fine-tune BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZUkXhM1k_TAl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 8\n",
        "EPOCHS = 10"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jGwV0yqg_o2u",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        },
        "outputId": "ba896231-4959-4df2-9c85-c2004ecfa758"
      },
      "source": [
        "train_tokens_tensor = torch.tensor(train_tokens_ids)\n",
        "train_y_tensor = torch.tensor(train_y.reshape(-1, 1)).float()\n",
        "\n",
        "test_tokens_tensor = torch.tensor(test_tokens_ids)\n",
        "test_y_tensor = torch.tensor(test_y.reshape(-1, 1)).float()\n",
        "\n",
        "train_masks_tensor = train_masks\n",
        "test_masks_tensor = test_masks\n",
        "\n",
        "str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  after removing the cwd from sys.path.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'439.065088M'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Yl2JpCe9YAu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_dataset = TensorDataset(train_tokens_tensor, train_masks_tensor, train_y_tensor)\n",
        "train_sampler = RandomSampler(train_dataset)\n",
        "train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=BATCH_SIZE)\n",
        "\n",
        "test_dataset = TensorDataset(test_tokens_tensor, test_masks_tensor, test_y_tensor)\n",
        "test_sampler = SequentialSampler(test_dataset)\n",
        "test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=BATCH_SIZE)\n"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JF_QD0naS8EQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "param_optimizer = list(bert_clf.sigmoid.named_parameters()) \n",
        "optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b28PcoDh_cyd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = Adam(bert_clf.parameters(), lr=3e-6)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z6yIChYvBP5F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " torch.cuda.empty_cache()"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqh8tCl4AFjo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 538
        },
        "outputId": "7bffdc11-5ae6-48f6-ff62-acced4625df3"
      },
      "source": [
        "for epoch_num in range(EPOCHS):\n",
        "    bert_clf.train()\n",
        "    train_loss = 0\n",
        "    for step_num, batch_data in enumerate(train_dataloader):\n",
        "        token_ids, masks, labels = tuple(t.to(device) for t in batch_data)\n",
        "        #print(str(torch.cuda.memory_allocated(device)/1000000 ) + 'M')\n",
        "        logits = bert_clf(token_ids, masks)\n",
        "        \n",
        "        loss_func = nn.BCELoss()\n",
        "\n",
        "        batch_loss = loss_func(logits, labels)\n",
        "        train_loss += batch_loss.item()\n",
        "        \n",
        "        \n",
        "        bert_clf.zero_grad()\n",
        "        batch_loss.backward()\n",
        "        \n",
        "\n",
        "        clip_grad_norm_(parameters=bert_clf.parameters(), max_norm=1.0)\n",
        "        optimizer.step()\n",
        "        \n",
        "        #clear_output(wait=True)\n",
        "      \n",
        "    print('Epoch: ', epoch_num + 1)\n",
        "    print(\"{0}/{1} train loss: {2} \".format(step_num, len(train_data) / BATCH_SIZE, train_loss / (step_num + 1)))\n",
        "\n",
        "    bert_clf.eval()\n",
        "    bert_predicted = []\n",
        "    all_logits = []\n",
        "    test_loss = 0\n",
        "    with torch.no_grad():\n",
        "        for step_num_e, batch_data in enumerate(test_dataloader):\n",
        "          \n",
        "            token_ids, masks, labels = tuple(t.to(device) for t in batch_data)\n",
        "\n",
        "            logits = bert_clf(token_ids, masks)\n",
        "            loss_func = nn.BCELoss()\n",
        "            loss = loss_func(logits, labels)\n",
        "            test_loss += loss.item()\n",
        "            numpy_logits = logits.cpu().detach().numpy()\n",
        "        \n",
        "            bert_predicted += list(numpy_logits[:, 0] > 0.5)\n",
        "            all_logits += list(numpy_logits[:, 0])\n",
        "\n",
        "    print(\"{0}/{1} val loss: {2} \".format(step_num_e, len(test_data) / BATCH_SIZE, test_loss / (step_num_e + 1)))\n",
        "        "
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch:  1\n",
            "124/125.0 train loss: 0.659061752319336 \n",
            "12/12.5 val loss: 0.6069660232617304 \n",
            "Epoch:  2\n",
            "124/125.0 train loss: 0.5017909047603607 \n",
            "12/12.5 val loss: 0.4374066568337954 \n",
            "Epoch:  3\n",
            "124/125.0 train loss: 0.3494661375284195 \n",
            "12/12.5 val loss: 0.38035118121367234 \n",
            "Epoch:  4\n",
            "124/125.0 train loss: 0.26325351601839064 \n",
            "12/12.5 val loss: 0.37102163239167285 \n",
            "Epoch:  5\n",
            "124/125.0 train loss: 0.19409464579820632 \n",
            "12/12.5 val loss: 0.3402688652276993 \n",
            "Epoch:  6\n",
            "124/125.0 train loss: 0.14372980040311814 \n",
            "12/12.5 val loss: 0.41000609672986543 \n",
            "Epoch:  7\n",
            "124/125.0 train loss: 0.09667221650481224 \n",
            "12/12.5 val loss: 0.43049231744729555 \n",
            "Epoch:  8\n",
            "124/125.0 train loss: 0.06809147936105728 \n",
            "12/12.5 val loss: 0.5220695082098246 \n",
            "Epoch:  9\n",
            "124/125.0 train loss: 0.047498719967901705 \n",
            "12/12.5 val loss: 0.5689449670652931 \n",
            "Epoch:  10\n",
            "124/125.0 train loss: 0.032736159820109606 \n",
            "12/12.5 val loss: 0.6105231463980789 \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GHFWhkRYHv5l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bert_clf.eval()\n",
        "bert_predicted = []\n",
        "all_logits = []\n",
        "with torch.no_grad():\n",
        "    for step_num, batch_data in enumerate(test_dataloader):\n",
        "\n",
        "        token_ids, masks, labels = tuple(t.to(device) for t in batch_data)\n",
        "\n",
        "        logits = bert_clf(token_ids, masks)\n",
        "        loss_func = nn.BCELoss()\n",
        "        loss = loss_func(logits, labels)\n",
        "        numpy_logits = logits.cpu().detach().numpy()\n",
        "        \n",
        "        bert_predicted += list(numpy_logits[:, 0] > 0.5)\n",
        "        all_logits += list(numpy_logits[:, 0])\n",
        "    "
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vg_sX9BjooL-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d502f80f-18e1-4268-f980-8b41e9948548"
      },
      "source": [
        "np.mean(bert_predicted)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.56"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-DmIJqUnkVM8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "4ebb5647-fd14-4f64-8125-55dd1ba0c918"
      },
      "source": [
        "print(classification_report(test_y, bert_predicted))"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "       False       0.93      0.82      0.87        50\n",
            "        True       0.84      0.94      0.89        50\n",
            "\n",
            "    accuracy                           0.88       100\n",
            "   macro avg       0.89      0.88      0.88       100\n",
            "weighted avg       0.89      0.88      0.88       100\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eIBvoExLpOne",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KQTaq4CdhU_9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "5de2b84a-87ea-4961-83ab-d73638a10442"
      },
      "source": [
        "for i in range(5):\n",
        "  clear_output(wait=True)\n",
        "  print('a', i+1)\n",
        "  print(\"{0}/{1} loss: {2} \".format(i, i / 5, i / (i + 1)))"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "a 5\n",
            "4/0.8 loss: 0.8 \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7h_IphSRhVCc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v0unLLybhVFg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 39,
      "outputs": []
    }
  ]
}