Train-GPT2-Multiclass.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9SDVhe1mpqPo",
        "outputId": "3b2da0c7-1412-4383-85c2-5415ea53188d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting accelerate\n",
            "  Downloading accelerate-0.24.1-py3-none-any.whl (261 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.2)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.1.0+cu118)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.19.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.1.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (4.66.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2023.7.22)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
            "Installing collected packages: accelerate\n",
            "Successfully installed accelerate-0.24.1\n",
            "Requirement already satisfied: transformers[torch] in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.19.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.4.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (4.66.1)\n",
            "Requirement already satisfied: torch!=1.12.0,>=1.10 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.1.0+cu118)\n",
            "Requirement already satisfied: accelerate>=0.20.3 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.24.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.20.3->transformers[torch]) (5.9.5)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers[torch]) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers[torch]) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (3.1.2)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (2.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2023.7.22)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch!=1.12.0,>=1.10->transformers[torch]) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch!=1.12.0,>=1.10->transformers[torch]) (1.3.0)\n"
          ]
        }
      ],
      "source": [
        "! pip install accelerate -U\n",
        "! pip install transformers[torch]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EPGgMyXWsQXQ",
        "outputId": "f17b710c-ef92-4823-8725-270ae56280d6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at FpOliveira/tupi-gpt2-small and are newly initialized because the shapes did not match:\n",
            "- score.weight: found shape torch.Size([2, 768]) in the checkpoint and torch.Size([12, 768]) in the model instantiated\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "432\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "from transformers import Trainer, TrainingArguments\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "from transformers import AdamW, get_linear_schedule_with_warmup\n",
        "import pandas as pd\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from sklearn.model_selection import train_test_split\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "\n",
        "# Enable GPU\n",
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "model_name = \"tupi-bert-large-portuguese-cased-multiclass-multilabel\"\n",
        "model_tokenizer_path = f\"FpOliveira/{model_name}\"\n",
        "\n",
        "# Define features and target\n",
        "x_name = \"text\"\n",
        "label_columns = ['ageism', 'aporophobia', 'body_shame', 'capacitism', 'lgbtphobia', 'political',\n",
        "                 'racism', 'religious_intolerance', 'misogyny', 'xenophobia', 'other']\n",
        "\n",
        "\n",
        "# Load and split the dataset into training and validation sets (stratified)\n",
        "df = pd.read_csv(\"https://raw.githubusercontent.com/Silly-Machine/TuPi-Portuguese-Hate-Speech-Dataset/main/datasets/tupi_hierarchy.csv\")\n",
        "df['not_hate'] = df[label_columns].apply(lambda row: 1 if row.sum() == 0 else 0, axis=1)\n",
        "label_columns.append('not_hate')\n",
        "df = df[['text']+label_columns]\n",
        "train_texts, val_texts, train_labels, val_labels = train_test_split(df[x_name], df[label_columns], test_size=0.2, random_state=42, stratify=df['not_hate'])\n",
        "test_dataset = pd.DataFrame({x_name: val_texts})\n",
        "test_dataset[label_columns] = pd.DataFrame(val_labels[label_columns].values.tolist(), index=test_dataset.index)\n",
        "\n",
        "# Define the task\n",
        "task_name = \"hate_multi_label_classification\"\n",
        "num_labels = len(labels_list)\n",
        "\n",
        "# Load the tokenizer\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"FpOliveira/tupi-gpt2-small\")\n",
        "\n",
        "# Load the pre-trained GPT-3 model\n",
        "model = AutoModelForSequenceClassification.from_pretrained(\"FpOliveira/tupi-gpt2-small\",num_labels=len(labels_list),ignore_mismatched_sizes=True)\n",
        "\n",
        "# Add a classification head on top of the model\n",
        "# model.resize_token_embeddings(len(tokenizer))\n",
        "# model.classifier = torch.nn.Linear(model.config.hidden_size, num_labels)\n",
        "\n",
        "# set the pad token in the tokenizer and model\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "model.config.pad_token_id = model.config.eos_token_id\n",
        "\n",
        "train_encodings = tokenizer(list(train_texts), truncation=True, padding=True)\n",
        "val_encodings = tokenizer(list(val_texts), truncation=True, padding=True)\n",
        "\n",
        "train_labels = torch.tensor(train_labels.values)\n",
        "val_labels = torch.tensor(val_labels.values)\n",
        "\n",
        "# Check tokenization output\n",
        "# Check tokenization output\n",
        "print(len(train_encodings['input_ids'][0]))\n",
        "\n",
        "class NewsDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "train_dataset = NewsDataset(train_encodings, train_labels)\n",
        "val_dataset = NewsDataset(val_encodings, val_labels)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Adjust the model's output layer\n",
        "model.classifier = torch.nn.Linear(model.config.hidden_size, num_labels)\n",
        "\n",
        "# Define training parameters\n",
        "optimizer = AdamW(model.parameters(), lr=1e-5)\n",
        "num_epochs = 3\n",
        "num_warmup_steps = 0  # You may adjust this based on your specific requirements\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=len(train_dataset) * num_epochs)\n",
        "\n",
        "# Training loop\n",
        "torch.cuda.empty_cache()\n",
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "model.to(device)\n",
        "model.train()\n",
        "\n",
        "for epoch in range(num_epochs):\n",
        "    print(f\"\\nEpoch {epoch + 1}/{num_epochs}\")\n",
        "\n",
        "    train_dataloader = DataLoader(train_dataset, batch_size=38, shuffle=True)\n",
        "\n",
        "    total_batches = len(train_dataloader)\n",
        "    total_loss = 0.0\n",
        "\n",
        "    for batch_idx, batch in enumerate(train_dataloader, 1):\n",
        "        optimizer.zero_grad()\n",
        "        input_ids = batch['input_ids'].to(device)\n",
        "        attention_mask = batch['attention_mask'].to(device)\n",
        "        labels = batch['labels'].to(device)\n",
        "\n",
        "        outputs = model(input_ids, attention_mask=attention_mask)\n",
        "\n",
        "        # Assuming your model returns logits, not loss directly\n",
        "        logits = outputs.logits\n",
        "\n",
        "        # Use BCEWithLogitsLoss for multi-label classification\n",
        "        loss_fn = torch.nn.BCEWithLogitsLoss()\n",
        "        loss = loss_fn(logits, labels.float())  # Convert labels to float\n",
        "\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "\n",
        "        total_loss += loss.item()\n",
        "\n",
        "        if batch_idx % 100 == 0:\n",
        "            print(f\"Batch {batch_idx}/{total_batches}, Loss: {loss.item()}\")\n",
        "\n",
        "    average_loss = total_loss / total_batches\n",
        "    print(f\"Epoch {epoch + 1} - Average Loss: {average_loss}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3solfM6eb5lF",
        "outputId": "06ecfd02-3dd2-4001-b556-08f6b79e6bde"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Epoch 1/3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-5-e57d576a215c>:59: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item['labels'] = torch.tensor(self.labels[idx])\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch 100/920, Loss: 0.11196088790893555\n",
            "Batch 200/920, Loss: 0.08852661401033401\n",
            "Batch 300/920, Loss: 0.11607339233160019\n",
            "Batch 400/920, Loss: 0.0970134437084198\n",
            "Batch 500/920, Loss: 0.03887900337576866\n",
            "Batch 600/920, Loss: 0.13669031858444214\n",
            "Batch 700/920, Loss: 0.04247033968567848\n",
            "Batch 800/920, Loss: 0.0601988285779953\n",
            "Batch 900/920, Loss: 0.028390266001224518\n",
            "Epoch 1 - Average Loss: 0.10353214330161395\n",
            "\n",
            "Epoch 2/3\n",
            "Batch 100/920, Loss: 0.08226053416728973\n",
            "Batch 200/920, Loss: 0.08628883957862854\n",
            "Batch 300/920, Loss: 0.03594090789556503\n",
            "Batch 400/920, Loss: 0.07123199850320816\n",
            "Batch 500/920, Loss: 0.08268043398857117\n",
            "Batch 600/920, Loss: 0.048679694533348083\n",
            "Batch 700/920, Loss: 0.05581311509013176\n",
            "Batch 800/920, Loss: 0.07852859795093536\n",
            "Batch 900/920, Loss: 0.08576615899801254\n",
            "Epoch 2 - Average Loss: 0.07392872004648265\n",
            "\n",
            "Epoch 3/3\n",
            "Batch 100/920, Loss: 0.0466112457215786\n",
            "Batch 200/920, Loss: 0.038809917867183685\n",
            "Batch 300/920, Loss: 0.1086466982960701\n",
            "Batch 400/920, Loss: 0.10912928730249405\n",
            "Batch 500/920, Loss: 0.07474884390830994\n",
            "Batch 600/920, Loss: 0.03519603610038757\n",
            "Batch 700/920, Loss: 0.08439010381698608\n",
            "Batch 800/920, Loss: 0.07246467471122742\n",
            "Batch 900/920, Loss: 0.05479725822806358\n",
            "Epoch 3 - Average Loss: 0.06755986663553377\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "_B_4RY_6daPB"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": [],
      "gpuType": "A100"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}