dataset info icd10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "from pprint import pprint\n",
    "\n",
    "import evaluate\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch import nn\n",
    "from datasets import load_dataset\n",
    "from peft import LoraConfig, TaskType, get_peft_model\n",
    "from transformers import (\n",
    "    AutoConfig,\n",
    "    AutoTokenizer,\n",
    "    AutoModelForSequenceClassification,\n",
    "    DataCollatorWithPadding,\n",
    "    EvalPrediction,\n",
    "    OPTForSequenceClassification,\n",
    "    Trainer,\n",
    "    TrainingArguments,\n",
    ")\n",
    "\n",
    "MODEL = \"emilyalsentzer/Bio_ClinicalBERT\"\n",
    "MAX_POSITION_EMBEDDINGS = 512\n",
    "\n",
    "from dataclasses import dataclass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DATSET_PATH = \"./data/train_10_top50.csv\"\n",
    "VAL_DATASET_PATH = \"./data/val_10_top50.csv\"\n",
    "TEST_DATSET_PATH = \"./data/test_10_top50.csv\"\n",
    "CODE_PATH = \"./data/icd10_codes_top50.csv\"\n",
    "\n",
    "# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device = \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "356f3062064f46fe9ea219801efea004",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/33768 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e26653bf5654a12977a701199f31246",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/4221 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b0ba3d0b70844ce80c629276f7ab7b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/4221 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load dataset\n",
    "# Load dataset\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL, use_fast=True, device=device)\n",
    "\n",
    "data_files = {\n",
    "    \"train\": TRAIN_DATSET_PATH,\n",
    "    \"validation\": VAL_DATASET_PATH,\n",
    "    \"test\": TEST_DATSET_PATH,\n",
    "}\n",
    "code_labels = pd.read_csv(CODE_PATH)\n",
    "dataset = load_dataset(\"csv\", data_files=data_files)\n",
    "# Create class dictionaries\n",
    "classes = [class_ for class_ in code_labels[\"icd_code\"] if class_]\n",
    "class2id = {class_: id for id, class_ in enumerate(classes)}\n",
    "id2class = {id: class_ for class_, id in class2id.items()}\n",
    "\n",
    "\n",
    "def multi_labels_to_ids(labels: list[str]) -> list[float]:\n",
    "    ids = [0.0] * len(class2id)  # BCELoss requires float as target type\n",
    "    for label in labels:\n",
    "        ids[class2id[label]] = 1.0\n",
    "    return ids\n",
    "\n",
    "\n",
    "def preprocess_function(example):\n",
    "    result = tokenizer(\n",
    "        example[\"text\"]\n",
    "    )\n",
    "    result[\"label\"] = [multi_labels_to_ids(eval(label)) for label in example[\"label\"]]\n",
    "    return result\n",
    "\n",
    "\n",
    "dataset = dataset.map(\n",
    "    preprocess_function, batched=True, num_proc=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "    num_rows: 33768\n",
       "})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[\"train\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "# get average length of text\n",
    "train_dataset = dataset['train']\n",
    "\n",
    "\n",
    "input_ids_lengths =[]\n",
    "label_lengths = []\n",
    "sets = ['train', 'test', 'validation']\n",
    "for set in sets:\n",
    "    input_ids_lengths.extend([len(example['input_ids']) for example in dataset[set]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3041.86259180289\n",
      "6262\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(input_ids_lengths))\n",
    "print(np.max(input_ids_lengths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_lengths = []\n",
    "\n",
    "sets = [\"train\", \"test\", \"validation\"]\n",
    "for set in sets:\n",
    "    label_lengths.extend([example['label'].count(1) for example in dataset[set]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.023146173892442\n",
      "23\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(label_lengths))\n",
    "print(np.max(label_lengths))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}