project/misc_pytorch_fastai_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datasets\n",
    "\n",
    "Datasets are `Iterable` (through their `__getitem__` and `__len__` attribute).\n",
    "Datasets are provided to `DataLoaders` which perform the aggreation to batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pimmslearn.io.datasplits import long_format\n",
    "from fastai.collab import CollabDataLoaders\n",
    "import random\n",
    "\n",
    "import pandas as pd\n",
    "import pimmslearn.io.datasets as datasets\n",
    "import pimmslearn.utils as test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N, M = 15, 7\n",
    "data = test_data.create_random_missing_data(N, M, prop_missing=.4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datasets\n",
    "\n",
    "- `DatasetWithMaskAndNoTarget`\n",
    "- `DatasetWithTarget`\n",
    "- `DatasetWithTargetSpecifyTarget`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### `DatasetWithMaskAndNoTarget`\n",
    "- base class for datasets with missing values and no target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = datasets.DatasetWithMaskAndNoTarget(df=pd.DataFrame(data))\n",
    "for _mask, _array in dataset:\n",
    "    break\n",
    "_array, _mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `DatasetWithTarget`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = test_data.create_random_missing_data(N, M, prop_missing=0.3)\n",
    "dataset = datasets.DatasetWithTarget(df=pd.DataFrame(data))\n",
    "\n",
    "for _mask, _array, target in dataset:\n",
    "    if any(_mask):\n",
    "        print(_array, _mask, target, sep='\\n')\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `DatasetWithTargetSpecifyTarget`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = test_data.create_random_missing_data(N, M, prop_missing=0.2)\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "val_y = df.stack().groupby(level=0).sample(frac=0.2)\n",
    "targets = val_y.unstack().sort_index(axis=1)\n",
    "\n",
    "df[targets.notna()] = pd.NA\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The targets are complementary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = datasets.DatasetWithTargetSpecifyTarget(df=df, targets=targets)\n",
    "for _mask, _array, target in dataset:\n",
    "    if any(_mask):\n",
    "        print(_mask, _array, target, sep='\\n')\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "row = random.randint(0, len(dataset) - 1)\n",
    "print(f\"{row = }\")\n",
    "dataset[row]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37ba1a81",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "dataset[row:row + 2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DataLoaders\n",
    "\n",
    "FastAI DataLoaders accept pytorch datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# , MSELossFlat, Learner\n",
    "# from fastai.collab import EmbeddingDotBias\n",
    "\n",
    "\n",
    "data = pd.DataFrame(data)\n",
    "data.index.name, data.columns.name = ('Sample ID', 'peptide')\n",
    "df_long = long_format(pd.DataFrame(data))\n",
    "df_long.reset_index(inplace=True)\n",
    "df_long.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = CollabDataLoaders.from_df(df_long, valid_pct=0.15,\n",
    "                                user_name='Sample ID', item_name='peptide', rating_name='intensity',\n",
    "                                bs=4)\n",
    "type(dls.dataset), dls.dataset._dl_type  # no __mro__?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Iterating over the dataset gives the column names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x in dls.dataset:\n",
    "    print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training DataFrame is hidden under items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls.dataset.items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x in dls.train_ds:\n",
    "    print(x)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls.train_ds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Iterating over the dataset returns columns, not single rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls.train_ds.__getitem__??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls.train_ds.items['Sample ID']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But the `DataLoader` return the numeric representation in batches:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch in dls.train_ds:\n",
    "    break\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls.train.__iter__??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_SingleProcessDataLoaderIter??"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So.. It seems too complicated\n",
    "- the `_collate_fn` seems to aggrete the data from the DataFrame\n",
    "- should be possible to keep track of that"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "next(iter(dls.dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "ca718f398b3a596c3df6787ca2afa269ec54c58eb9478d66aeb41db8e6cb8262"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}