part-2-bert-model-for-comment-classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Toxic Comment Classification THE TRAINING"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PART II"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import random\n",
    "import time\n",
    "\n",
    "import re\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set(style=\"ticks\",context=\"talk\")\n",
    "plt.style.use('dark_background')\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as func\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "import transformers\n",
    "from transformers import AdamW, get_linear_schedule_with_warmup\n",
    "\n",
    "import tokenizers\n",
    "from sklearn.metrics import mean_squared_error, roc_auc_score, roc_curve, auc\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "SEED = 34\n",
    "\n",
    "def random_seed(SEED):\n",
    "    random.seed(SEED)\n",
    "    os.environ['PYTHONHASHSEED'] = str(SEED)\n",
    "    np.random.seed(SEED)\n",
    "    torch.manual_seed(SEED)\n",
    "    torch.cuda.manual_seed(SEED)\n",
    "    torch.cuda.manual_seed_all(SEED)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "\n",
    "random_seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('../Toxic-Comment-Classification/input/train.csv', nrows = 200 )\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "temp = train[train['toxic'] == 1]\n",
    "temp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(len(train['comment_text'][10]), 'Total Characters')\n",
    "train['comment_text'][10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "labels = train.drop(['id', 'comment_text'], axis = 1)\n",
    "unique_values = lambda x: train[x].unique()\n",
    "[unique_values(col) for col in labels.columns.tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv('../Toxic-Comment-Classification/input/train.csv', nrows = 10)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_labels = pd.read_csv('../Toxic-Comment-Classification/input/test_labels.csv', nrows = 10)\n",
    "test_labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission = pd.read_csv('../Toxic-Comment-Classification/input/sample_submission.csv', nrows = 10)\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = train.drop(['id', 'comment_text'], axis = 1)\n",
    "label_counts = df_train.sum()\n",
    "df_counts = pd.DataFrame(label_counts)\n",
    "df_counts.rename(columns = {0:'counts'}, inplace = True)\n",
    "df_counts = df_counts.sort_values('counts', ascending = False)\n",
    "df_counts "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "\n",
    "    text = re.sub('\\[.*?\\]', '', text)\n",
    "    #pattern = [zero or more character]\n",
    "\n",
    "    text = re.sub('https?://\\S+|www\\.\\S+', '', text)\n",
    "    #pattern = removes (http),://, 'and' www.\n",
    "    \n",
    "    text = re.sub('<.*?>+', '', text)\n",
    "    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)\n",
    "    #pattern = any punctionation\n",
    "\n",
    "    text = re.sub('\\n', '', text)\n",
    "    #pattern = any new line\n",
    "\n",
    "    text = re.sub('\\w*\\d\\w*', '', text)\n",
    "    #pattern = any from[a-zA-Z0-9_], any from[0-9], any from [a-zA-Z0-9_]\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "train['clean_text'] = train['comment_text'].apply(str).apply(lambda x: clean_text(x))\n",
    "test['clean_text'] = test['comment_text'].apply(str).apply(lambda x: clean_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = 5\n",
    "train['kfold'] = train.index % kfold\n",
    "train.index % kfold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_train = train[train[\"kfold\"] != 0].reset_index(drop = True)\n",
    "p_valid = train[train[\"kfold\"] == 0].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "senten_len = []\n",
    "#tqdm is progress bar\n",
    "for sentence in tqdm(p_train['clean_text']):\n",
    "    token_words = tokenizer.encode_plus(sentence)['input_ids']\n",
    "    senten_len.append(len(token_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len = 256"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We define a class BertDataSet with Dataset as super class and overwirte the __init__, __len__ and __getitem__ function in it. It will get the comment list and relevant toxic labels (6 labels in this case) and creates token ids and attention mask to distinguish the comments from the zero padding.\n",
    "\n",
    "\n",
    "torch.tensors are designed to be used in the context of gradient descent optimization, and therefore they hold not only a tensor with numeric values, but (and more importantly) the computational graph leading to these values. This computational graph is then used (using the chain rule of derivatives) to compute the derivative of the loss function w.r.t each of the independent variables used to compute the loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BertDataSet(Dataset):\n",
    "#Bidirectional Encoder Representations from Transformers\n",
    "    \n",
    "    def __init__(self, sentences, toxic_labels):\n",
    "        self.sentences = sentences\n",
    "        #target is a matrix with shape [#1 x #6(toxic, obscene, etc)]\n",
    "        self.targets = toxic_labels.to_numpy()\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.sentences)\n",
    "    \n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        sentence = self.sentences[idx]\n",
    "        bert_senten = tokenizer.encode_plus(sentence, \n",
    "                                            add_special_tokens = True, # [CLS],[SEP]\n",
    "                                            max_length = max_len,\n",
    "                                            pad_to_max_length = True,\n",
    "                                            truncation = True,\n",
    "                                            return_attention_mask = True\n",
    "                                             )\n",
    "        ids = torch.tensor(bert_senten['input_ids'], dtype = torch.long)\n",
    "        mask = torch.tensor(bert_senten['attention_mask'], dtype = torch.long)\n",
    "        toxic_label = torch.tensor(self.targets[idx], dtype = torch.float)\n",
    "        \n",
    "        \n",
    "        return {\n",
    "            'ids' : ids,\n",
    "            'mask' : mask,\n",
    "            'toxic_label':toxic_label\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = BertDataSet(p_train['clean_text'], p_train[['toxic', 'severe_toxic','obscene', 'threat', 'insult','identity_hate']])\n",
    "valid_dataset = BertDataSet(p_valid['clean_text'], p_valid[['toxic', 'severe_toxic','obscene', 'threat', 'insult','identity_hate']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for a in train_dataset:\n",
    "#     print(a)\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batch = 32\n",
    "valid_batch = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader = DataLoader(train_dataset, batch_size = train_batch, pin_memory = True, num_workers = 4, shuffle = True)\n",
    "valid_dataloader = DataLoader(valid_dataset, batch_size = valid_batch, pin_memory = True, num_workers = 4, shuffle = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "model = transformers.BertForSequenceClassification.from_pretrained('bert-base-cased', num_labels = 6)\n",
    "model.to(device)\n",
    "model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "for a in train_dataloader:\n",
    "    ids = a['ids'].to(device)\n",
    "    mask = a['mask'].to(device)\n",
    "    output = model(ids, mask)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "func.softmax(output['logits'], dim = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_probs = func.softmax(output['logits'], dim = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.max(output_probs, dim = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 5\n",
    "LR = 2e-5 #Learning rate\n",
    "optimizer = AdamW(model.parameters(), LR, betas = (0.9, 0.999), weight_decay = 1e-2, correct_bias = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_steps = int((len(train) * epochs)/train_batch)\n",
    "num_steps = int(train_steps * 0.1)\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, num_steps, train_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = nn.BCEWithLogitsLoss()\n",
    "loss_fn.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = torch.cuda.amp.GradScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training(train_dataloader, model, optimizer, scheduler):\n",
    "    model.train()\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "    correct_predictions = 0\n",
    "    \n",
    "    for a in train_dataloader:\n",
    "        losses = []\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        #allpreds = []\n",
    "        #alltargets = []\n",
    "        \n",
    "        with torch.cuda.amp.autocast():\n",
    "            \n",
    "            ids = a['ids'].to(device, non_blocking = True)\n",
    "            mask = a['mask'].to(device, non_blocking = True) \n",
    "\n",
    "            output = model(ids, mask) #This gives model as output, however we want the values at the output\n",
    "            output = output['logits'].squeeze(-1).to(torch.float32)\n",
    "\n",
    "            output_probs = torch.sigmoid(output)\n",
    "            preds = torch.where(output_probs > 0.5, 1, 0)\n",
    "            \n",
    "            toxic_label = a['toxic_label'].to(device, non_blocking = True) \n",
    "            loss = loss_fn(output, toxic_label)            \n",
    "            \n",
    "            losses.append(loss.item())\n",
    "            #allpreds.append(output.detach().cpu().numpy())\n",
    "            #alltargets.append(toxic.detach().squeeze(-1).cpu().numpy())\n",
    "            correct_predictions += torch.sum(preds == toxic_label)\n",
    "        \n",
    "        scaler.scale(loss).backward() #Multiplies (‘scales’) a tensor or list of tensors by the scale factor.\n",
    "                                      #Returns scaled outputs. If this instance of GradScaler is not enabled, outputs are returned unmodified.\n",
    "        scaler.step(optimizer) #Returns the return value of optimizer.step(*args, **kwargs).\n",
    "        scaler.update() #Updates the scale factor.If any optimizer steps were skipped the scale is multiplied by backoff_factor to reduce it. \n",
    "                        #If growth_interval unskipped iterations occurred consecutively, the scale is multiplied by growth_factor to increase it\n",
    "        scheduler.step() # Update learning rate schedule\n",
    "    \n",
    "    losses = np.mean(losses)\n",
    "    corr_preds = correct_predictions.detach().cpu().numpy()\n",
    "    accuracy = corr_preds/(len(p_train)*6)\n",
    "    \n",
    "    return losses, accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validating(valid_dataloader, model):\n",
    "    \n",
    "    model.eval()\n",
    "    correct_predictions = 0\n",
    "    all_output_probs = []\n",
    "    \n",
    "    for a in valid_dataloader:\n",
    "        losses = []\n",
    "        ids = a['ids'].to(device, non_blocking = True)\n",
    "        mask = a['mask'].to(device, non_blocking = True)\n",
    "        output = model(ids, mask)\n",
    "        output = output['logits'].squeeze(-1).to(torch.float32)\n",
    "        output_probs = torch.sigmoid(output)\n",
    "        preds = torch.where(output_probs > 0.5, 1, 0)\n",
    "            \n",
    "        toxic_label = a['toxic_label'].to(device, non_blocking = True)\n",
    "        loss = loss_fn(output, toxic_label)\n",
    "        losses.append(loss.item())\n",
    "        all_output_probs.extend(output_probs.detach().cpu().numpy())\n",
    "        \n",
    "        correct_predictions += torch.sum(preds == toxic_label)\n",
    "        corr_preds = correct_predictions.detach().cpu().numpy()\n",
    "    \n",
    "    losses = np.mean(losses)\n",
    "    corr_preds = correct_predictions.detach().cpu().numpy()\n",
    "    accuracy = corr_preds/(len(p_valid)*6)\n",
    "    \n",
    "    return losses, accuracy, all_output_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "best_score = 1000\n",
    "train_accs = []\n",
    "valid_accs = []\n",
    "train_losses = []\n",
    "valid_losses = []\n",
    "\n",
    "for eboch in tqdm(range(epochs)):\n",
    "    \n",
    "    train_loss, train_acc = training(train_dataloader, model, optimizer, scheduler)\n",
    "    valid_loss, valid_acc, valid_probs = validating(valid_dataloader, model)\n",
    "    \n",
    "    print('train losses: %.4f' % train_loss, 'train accuracy: %.3f' % train_acc)\n",
    "    print('valid losses: %.4f' % valid_loss, 'valid accuracy: %.3f' % valid_acc)\n",
    "    train_losses.append(train_loss)\n",
    "    valid_losses.append(valid_loss)\n",
    "    train_accs.append(train_acc)\n",
    "    valid_accs.append(valid_acc)\n",
    "    \n",
    "    \n",
    "    if valid_loss < best_score:\n",
    "        best_score = valid_loss\n",
    "        print('Found a good model!')\n",
    "        state = {\n",
    "            'state_dict': model.state_dict(),\n",
    "            'optimizer_dict': optimizer.state_dict(),\n",
    "            'best_score': best_score\n",
    "        }\n",
    "        torch.save(state, 'best_model.pth')\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.arange(epochs)\n",
    "fig, ax = plt.subplots(1, 2, figsize = (15,4))\n",
    "ax[0].plot(x, train_losses)\n",
    "ax[0].plot(x, valid_losses)\n",
    "ax[0].set_ylabel('Losses', weight = 'bold')\n",
    "ax[0].set_xlabel('Epochs')\n",
    "ax[0].grid(alpha = 0.3)\n",
    "ax[0].legend(labels = ['train losses', 'valid losses'])\n",
    "\n",
    "ax[1].plot(x, train_accs)\n",
    "ax[1].plot(x, valid_accs)\n",
    "ax[1].set_ylabel('Accuracy', weight = 'bold')\n",
    "ax[1].set_xlabel('Epochs')\n",
    "ax[1].legend(labels = ['train acc', 'valid acc'])\n",
    "\n",
    "ax[1].grid(alpha = 0.3)\n",
    "fig.suptitle('Fold = 0', weight = 'bold') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_loss, valid_acc, valid_probs = validating(valid_dataloader, model)\n",
    "valid_probs = np.asarray(valid_probs).flatten()\n",
    "y_valid = p_valid[['toxic', 'severe_toxic','obscene', 'threat', 'insult','identity_hate']].to_numpy().flatten()\n",
    "fpr, tpr, _ = roc_curve(y_valid, valid_probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "ax.plot(fpr, tpr)\n",
    "ax.set_title('ROC Curv')\n",
    "ax.set_xlabel('FPR')\n",
    "ax.set_ylabel('TPR')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc(fpr, tpr)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}