NLEMalariaMicrscopyFYP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "RobtMsbm0eiy"
      ],
      "toc_visible": true,
      "machine_shape": "hm",
      "gpuType": "T4",
      "authorship_tag": "ABX9TyNtSK1tjAeyyO3Dkj+/hu47",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/BRIAN12682/Automation-Projects/blob/main/NLEMalariaMicrscopyFYP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gy4N9xtZlQd5",
        "outputId": "c7f34118-5a0d-4664-b8e6-fcf6ea7fb563"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.41.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.14.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.23.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.12.25)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.4)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.0->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.0->transformers) (4.11.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.2.2)\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "znMthHJJyG76",
        "outputId": "0473c7e9-470b-4050-b0bb-65eb6a9e9401"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Updated folder paths\n",
        "img_dir = '/content/drive/MyDrive/DATASETS/MalariaPI/images'\n",
        "annotation_dir = '/content/drive/MyDrive/DATASETS/MalariaPI/annotation'\n"
      ],
      "metadata": {
        "id": "ts3FlylSyauk"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "# Path to your annotations directory\n",
        "annotation_dir = '/content/drive/MyDrive/DATASETS/MalariaPI/annotation'\n",
        "\n",
        "# List XML files and select the first one\n",
        "xml_files = [f for f in os.listdir(annotation_dir) if f.endswith('.xml')]\n",
        "first_xml_file = xml_files[0]\n",
        "\n",
        "# Print the content of the first XML file\n",
        "with open(os.path.join(annotation_dir, first_xml_file), 'r') as file:\n",
        "    for i in range(10):  # Print the first 10 lines\n",
        "        print(file.readline().strip())\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jmpl-pmv3DBL",
        "outputId": "4791ab05-e42d-4e95-b375-336a6e8d1163"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<annotation>\n",
            "<source>\n",
            "<database>Makerere laboratory images database</database>\n",
            "<annotation>Makerere medical school/Mulago Hospital</annotation>\n",
            "<image></image>\n",
            "<flickrid></flickrid>\n",
            "</source>\n",
            "<owner>\n",
            "<flickrid></flickrid>\n",
            "<name>unknown</name>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import Dataset, DataLoader\n",
        "from PIL import Image\n",
        "import torchvision.transforms as transforms\n",
        "import os\n",
        "import xml.etree.ElementTree as ET\n",
        "import torch\n",
        "\n",
        "class MalariaDataset(Dataset):\n",
        "    def __init__(self, img_dir, annotation_dir, transform=None):\n",
        "        self.img_dir = img_dir\n",
        "        self.annotation_dir = annotation_dir\n",
        "        self.transform = transform\n",
        "        self.annotations = self.load_annotations()\n",
        "\n",
        "    def load_annotations(self):\n",
        "        annotations = []\n",
        "        for file in os.listdir(self.annotation_dir):\n",
        "            if file.endswith('.xml'):\n",
        "                tree = ET.parse(os.path.join(self.annotation_dir, file))\n",
        "                root = tree.getroot()\n",
        "                image_filename = file.replace('.xml', '.jpg')  # Assuming the image file matches the XML file name\n",
        "                image_path = os.path.join(self.img_dir, image_filename)\n",
        "                image = Image.open(image_path).convert('RGB')\n",
        "\n",
        "                objects = []\n",
        "                for obj in root.findall('object'):\n",
        "                    objects.append({\n",
        "                        'xmin': float(obj.find('bndbox/xmin').text),\n",
        "                        'ymin': float(obj.find('bndbox/ymin').text),\n",
        "                        'xmax': float(obj.find('bndbox/xmax').text),\n",
        "                        'ymax': float(obj.find('bndbox/ymax').text),\n",
        "                        'label': obj.find('name').text  # Adjust label as needed\n",
        "                    })\n",
        "\n",
        "                annotations.append({\n",
        "                    'image': image,\n",
        "                    'objects': objects,\n",
        "                    'infected': 1 if objects else 0  # 1 if there are pathogens, otherwise 0\n",
        "                })\n",
        "        return annotations\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.annotations)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        annotation = self.annotations[idx]\n",
        "        image = annotation['image']\n",
        "        if self.transform:\n",
        "            image = self.transform(image)\n",
        "\n",
        "        boxes = [[obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax']] for obj in annotation['objects']]\n",
        "        labels = [0 for _ in annotation['objects']]  # Assuming '0' for pathogen\n",
        "\n",
        "        target = {\n",
        "            'boxes': torch.tensor(boxes),\n",
        "            'labels': torch.tensor(labels),\n",
        "            'infected': torch.tensor(annotation['infected'])\n",
        "        }\n",
        "\n",
        "        return image, target\n",
        "\n",
        "# Define transformations\n",
        "transform = transforms.Compose([\n",
        "    transforms.Resize((224, 224)),\n",
        "    transforms.ToTensor(),\n",
        "])\n",
        "\n",
        "img_dir = '/content/drive/MyDrive/DATASETS/MalariaPI/images'\n",
        "annotation_dir = '/content/drive/MyDrive/DATASETS/MalariaPI/annotation'\n",
        "dataset = MalariaDataset(img_dir, annotation_dir, transform=transform)\n"
      ],
      "metadata": {
        "id": "F05qwbG5ymay"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data._utils.collate import default_collate\n",
        "\n",
        "def custom_collate_fn(batch):\n",
        "    # Separate data and targets\n",
        "    data = [item[0] for item in batch]\n",
        "    targets = [item[1] for item in batch]\n",
        "\n",
        "    # Use default collate for data (images)\n",
        "    data = default_collate(data)\n",
        "\n",
        "    # Targets need custom handling:\n",
        "    # Convert each attribute of targets to separate lists that can be processed individually\n",
        "    boxes = [target['boxes'] for target in targets]\n",
        "    labels = [target['labels'] for target in targets]\n",
        "    infected = torch.stack([target['infected'] for target in targets])\n",
        "\n",
        "    return data, {'boxes': boxes, 'labels': labels, 'infected': infected}\n"
      ],
      "metadata": {
        "id": "GCxAAAzg-d5Y"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_loader = DataLoader(dataset, batch_size=4, shuffle=True, collate_fn=custom_collate_fn)"
      ],
      "metadata": {
        "id": "pX5t3r8S960b"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from torchvision.models import vit_b_16\n",
        "import torch.optim as optim\n",
        "import torch.nn as nn\n",
        "\n",
        "# Assuming the Vision Transformer model is adapted to output both bounding boxes and classification\n",
        "model = vit_b_16(pretrained=True)\n",
        "# Adjust the classifier to output bounding boxes + 1 binary output for classification\n",
        "model.heads.head = nn.Linear(model.heads.head.in_features, 4 * len(dataset[0][1]['boxes']) + 1)\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "model.to(device)\n",
        "\n",
        "# Optimizer and loss function\n",
        "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
        "criterion = nn.BCEWithLogitsLoss()  # Change if using a different kind of target\n",
        "\n",
        "# Number of epochs\n",
        "epochs = 10\n",
        "for epoch in range(epochs):\n",
        "    model.train()\n",
        "    running_loss = 0.0\n",
        "    for images, targets in train_loader:\n",
        "        images = images.to(device)\n",
        "        infected_labels = targets['infected'].to(device)\n",
        "\n",
        "        # We need to handle variable-sized bounding boxes during the forward pass or calculate loss.\n",
        "        # Assuming your model and loss function can handle this variable input directly.\n",
        "        # If not, you might need to modify how you compute the loss.\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        outputs = model(images)\n",
        "        # Let's assume outputs are properly aligned with how you handle targets; adjust as necessary\n",
        "        predicted_infected = outputs[:, -1]  # Example, adjust index or method according to your model structure\n",
        "        loss_infected = criterion(predicted_infected, infected_labels.float())\n",
        "        # Add loss calculation for bounding boxes if necessary\n",
        "        loss = loss_infected\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        running_loss += loss.item()\n",
        "\n",
        "    print(f'Epoch {epoch + 1}, Loss: {running_loss / len(train_loader)}')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TZaMLb6U-EKS",
        "outputId": "2b2eae18-5925-45b6-9776-d7a3fe5156b2"
      },
      "execution_count": null,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ViT_B_16_Weights.IMAGENET1K_V1`. You can also use `weights=ViT_B_16_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "<ipython-input-31-07c111c9ce1b>:13: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  boxes = [torch.tensor(target['boxes']) for target in targets]\n",
            "<ipython-input-31-07c111c9ce1b>:14: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  labels = [torch.tensor(target['labels']) for target in targets]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from torchvision.models import vit_b_16, ViT_B_16_Weights\n",
        "\n",
        "\n",
        "# Load the pretrained Vision Transformer model with specified weights\n",
        "model = vit_b_16(weights=ViT_B_16_Weights.IMAGENET1K_V1)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VNljpPSpzRHn",
        "outputId": "53119702-1270-428e-9cc2-feefcde25d1a"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ViT_B_16_Weights.IMAGENET1K_V1`. You can also use `weights=ViT_B_16_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/vit_b_16-c867db91.pth\" to /root/.cache/torch/hub/checkpoints/vit_b_16-c867db91.pth\n",
            "100%|██████████| 330M/330M [00:02<00:00, 135MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from torchvision import transforms\n",
        "from PIL import Image\n",
        "import os\n",
        "\n",
        "class MalariaDataset(Dataset):\n",
        "    def __init__(self, folder_path, transform=None):\n",
        "        self.folder_path = folder_path\n",
        "        self.transform = transform\n",
        "        self.images = [f for f in os.listdir(folder_path) if f.endswith('.png')]  # Adjust based on your image file types\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.images)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        img_name = os.path.join(self.folder_path, self.images[idx])\n",
        "        image = Image.open(img_name)\n",
        "        if self.transform:\n",
        "            image = self.transform(image)\n",
        "        return image\n",
        "\n",
        "# Define transforms\n",
        "transform = transforms.Compose([\n",
        "    transforms.Resize((224, 224)),  # Resize to the input size expected by ViT\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # Normalization for pre-trained models\n",
        "])\n",
        "\n",
        "dataset = MalariaDataset(img_dir, transform=transform)\n",
        "dataloader = DataLoader(dataset, batch_size=32, shuffle=True)  # Adjust batch size based on your GPU memory\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 321
        },
        "id": "mkTfebh-0kQB",
        "outputId": "20273a43-7de1-499f-a81d-4dbfd2ecfa79"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "num_samples should be a positive integer value, but got num_samples=0",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-20-62b7a11f6f32>\u001b[0m in \u001b[0;36m<cell line: 31>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMalariaDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m \u001b[0mdataloader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Adjust batch size based on your GPU memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, dataset, batch_size, shuffle, sampler, batch_sampler, num_workers, collate_fn, pin_memory, drop_last, timeout, worker_init_fn, multiprocessing_context, generator, prefetch_factor, persistent_workers, pin_memory_device)\u001b[0m\n\u001b[1;32m    348\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# map-style\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    349\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 350\u001b[0;31m                     \u001b[0msampler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomSampler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgenerator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgenerator\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    351\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    352\u001b[0m                     \u001b[0msampler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSequentialSampler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/sampler.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data_source, replacement, num_samples, generator)\u001b[0m\n\u001b[1;32m    141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_samples\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 143\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"num_samples should be a positive integer value, but got num_samples={self.num_samples}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: num_samples should be a positive integer value, but got num_samples=0"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 32\n",
        "\n",
        "train_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 321
        },
        "id": "VoCBBXhJ0vs8",
        "outputId": "753d935b-c8ba-4cf8-c7e8-3b1d25e22685"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "num_samples should be a positive integer value, but got num_samples=0",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-18-b5a138870ae7>\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m32\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtrain_loader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, dataset, batch_size, shuffle, sampler, batch_sampler, num_workers, collate_fn, pin_memory, drop_last, timeout, worker_init_fn, multiprocessing_context, generator, prefetch_factor, persistent_workers, pin_memory_device)\u001b[0m\n\u001b[1;32m    348\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# map-style\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    349\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 350\u001b[0;31m                     \u001b[0msampler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomSampler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgenerator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgenerator\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    351\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    352\u001b[0m                     \u001b[0msampler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSequentialSampler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/sampler.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data_source, replacement, num_samples, generator)\u001b[0m\n\u001b[1;32m    141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_samples\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 143\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"num_samples should be a positive integer value, but got num_samples={self.num_samples}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: num_samples should be a positive integer value, but got num_samples=0"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# gpt-3"
      ],
      "metadata": {
        "id": "RobtMsbm0eiy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install openai"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vAZwe6IGzrUV",
        "outputId": "fc32b6a6-6d8b-4ec8-d74b-c5c69f8ffa3b"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting openai\n",
            "  Downloading openai-1.33.0-py3-none-any.whl (325 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m325.5/325.5 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai) (1.7.0)\n",
            "Collecting httpx<1,>=0.23.0 (from openai)\n",
            "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai) (2.7.3)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.1)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.4)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai) (4.12.1)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (2024.6.2)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai)\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (2.18.4)\n",
            "Installing collected packages: h11, httpcore, httpx, openai\n",
            "Successfully installed h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 openai-1.33.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import openai\n",
        "\n",
        "def generate_explanation(annotation, openai_api_key):\n",
        "    openai.api_key = openai_api_key\n",
        "\n",
        "    prompt = f\"Explain the significance of a malaria pathogen located at {annotation} in a microscopy image.\"\n",
        "    response = openai.Completion.create(\n",
        "        engine=\"text-davinci-002\",\n",
        "        prompt=prompt,\n",
        "        max_tokens=150\n",
        "    )\n",
        "    return response.choices[0].text\n",
        "\n",
        "# Example usage\n",
        "explanation = generate_explanation(annotations[0], 'your-openai-api-key')\n"
      ],
      "metadata": {
        "id": "qO_7p5Zezp6q"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}