project/01_1_transfer_NAGuideR_pred.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a75efcbe-2ae0-4609-872a-759fb5c80af1",
   "metadata": {},
   "source": [
    "# Transfer predictions from NAGuideR\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f29b93d1",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "import pimmslearn\n",
    "import pimmslearn.models\n",
    "import pimmslearn.pandas\n",
    "from pimmslearn.io import datasplits\n",
    "\n",
    "pimmslearn.plotting.make_large_descriptors(5)\n",
    "\n",
    "logger = pimmslearn.logging.setup_logger(logging.getLogger('pimmslearn'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbf23f02",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# catch passed parameters\n",
    "args = None\n",
    "args = dict(globals()).keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b02b864",
   "metadata": {},
   "source": [
    "Papermill script parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a08a34aa",
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# files and folders\n",
    "# Datasplit folder with data for experiment\n",
    "folder_experiment: str = 'runs/example'\n",
    "folder_data: str = ''  # specify data directory if needed\n",
    "file_format: str = 'csv'  # file format of create splits, default pickle (csv)\n",
    "identifer_str: str = '_all_'  # identifier for prediction files to be filtered\n",
    "dumps: list = None  # list of dumps to be used"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26815da7",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "Some argument transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d5c476b",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "args = pimmslearn.nb.get_params(args, globals=globals())\n",
    "args = pimmslearn.nb.args_from_dict(args)\n",
    "args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba3513a7",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "files_out = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba15118b",
   "metadata": {},
   "source": [
    "load data splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75341d2b",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data = datasplits.DataSplits.from_folder(\n",
    "    args.data, file_format=args.file_format)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbef8da3",
   "metadata": {},
   "source": [
    "Validation and test data split of simulated missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "723eacd2",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "val_pred_fake_na = data.val_y.to_frame(name='observed')\n",
    "val_pred_fake_na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "514d193f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "test_pred_fake_na = data.test_y.to_frame(name='observed')\n",
    "test_pred_fake_na.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "204838b7",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# Find and load prediction files, filter for validation and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c06dcd14",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if args.dumps is not None:\n",
    "    entire_pred = [Path(s) for s in args.dumps.split(',')]\n",
    "else:\n",
    "    entire_pred = list(file for file in args.out_preds.iterdir()\n",
    "                       if '_all_' in str(file))\n",
    "entire_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a9e66bc",
   "metadata": {
    "lines_to_next_cell": 0,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "mask = data.train_X.unstack().isna().stack()\n",
    "idx_real_na = mask.index[mask]\n",
    "idx_real_na = (idx_real_na\n",
    "               .drop(val_pred_fake_na.index)\n",
    "               .drop(test_pred_fake_na.index))\n",
    "\n",
    "for fpath in entire_pred:\n",
    "    logger.info(f\"Load {fpath = }\")\n",
    "    col_name = fpath.stem.split('_all_')[-1]\n",
    "    pred = pd.read_csv(fpath, index_col=[1, 0])\n",
    "    val_pred_fake_na[col_name] = pred\n",
    "    fname = args.out_preds / f'pred_val_{col_name}.csv'\n",
    "    files_out[fname.name] = fname.as_posix()\n",
    "    val_pred_fake_na[['observed', col_name]].to_csv(fname)\n",
    "    logger.info(f\"Save {fname = }\")\n",
    "\n",
    "    test_pred_fake_na[col_name] = pred\n",
    "    fname = args.out_preds / f'pred_test_{col_name}.csv'\n",
    "    files_out[fname.name] = fname.as_posix()\n",
    "    test_pred_fake_na[['observed', col_name]].to_csv(fname)\n",
    "    logger.info(f\"Save {fname = }\")\n",
    "    # hacky, but works:\n",
    "    pred_real_na = (pd.Series(0, index=idx_real_na, name='placeholder')\n",
    "                    .to_frame()\n",
    "                    .join(pred, how='left')\n",
    "                    .drop('placeholder', axis=1))\n",
    "    # pred_real_na.name = 'intensity'\n",
    "    fname = args.out_preds / f'pred_real_na_{col_name}.csv'\n",
    "    files_out[fname.name] = fname.as_posix()\n",
    "    pred_real_na.to_csv(fname)\n",
    "    logger.info(f\"Save {fname = }\")\n",
    "\n",
    "# del pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72adc8ec",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "val_pred_fake_na"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "655364c4",
   "metadata": {},
   "source": [
    "Metrics for simulated missing values (NA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7c5dab0",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# papermill_description=metrics\n",
    "d_metrics = pimmslearn.models.Metrics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "773dcbbe",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "added_metrics = d_metrics.add_metrics(val_pred_fake_na.dropna(how='all', axis=1), 'valid_fake_na')\n",
    "pd.DataFrame(added_metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33fde68c",
   "metadata": {},
   "source": [
    "## Test Datasplit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "065e1e62",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "added_metrics = d_metrics.add_metrics(test_pred_fake_na.dropna(how='all', axis=1), 'test_fake_na')\n",
    "pd.DataFrame(added_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37e8e515",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "metrics_df = pimmslearn.models.get_df_from_nested_dict(\n",
    "    d_metrics.metrics, column_levels=['model', 'metric_name']).T\n",
    "metrics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1f0d12c",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "order_methods = metrics_df.loc[pd.IndexSlice[:,\n",
    "                                             'MAE'], 'valid_fake_na'].sort_values()\n",
    "order_methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0f8ce7c",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "top_5 = ['observed', *order_methods.droplevel(-1).index[:6]]\n",
    "top_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e041594",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(8, 2))\n",
    "ax, errors_bind = pimmslearn.plotting.errors.plot_errors_binned(\n",
    "    val_pred_fake_na[top_5],\n",
    "    ax=ax,\n",
    ")\n",
    "fname = args.out_figures / 'NAGuideR_errors_per_bin_val.png'\n",
    "files_out[fname.name] = fname.as_posix()\n",
    "pimmslearn.savefig(ax.get_figure(), fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77b1b792",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "files_out"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "title,tags,-all"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}