project/02_4_best_models_over_all_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e5a0f3cf-c331-4f02-ab04-84827cd4c872",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Best model over datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dbabc1a9-6ede-4538-9998-6a821be076d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.express as px\n",
    "import pimmslearn.plotting\n",
    "import pimmslearn.nb\n",
    "\n",
    "\n",
    "pd.options.display.max_columns = 45\n",
    "pd.options.display.max_rows = 110\n",
    "pd.options.display.multi_sparse = False\n",
    "\n",
    "plt.rcParams['figure.figsize'] = [12.0, 6.0]\n",
    "\n",
    "pimmslearn.plotting.make_large_descriptors()\n",
    "\n",
    "logger = pimmslearn.logging.setup_nb_logger()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a76a848-1a52-42b8-b3b5-85320fbc8a0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Read input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ee3d67f4-7734-46fd-892f-8fc29f529d10",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_long = []\n",
    "\n",
    "for fname in snakemake.input:\n",
    "    print(f\"{fname = }\")\n",
    "    fname = Path(fname)\n",
    "    level = fname.parent.name\n",
    "\n",
    "    _df = pd.read_csv(fname)\n",
    "    _df['data level'] = level\n",
    "\n",
    "    metrics_long.append(_df)\n",
    "    del _df\n",
    "metrics_long = pd.concat(metrics_long)\n",
    "metrics_long['id'] = metrics_long['data level'] + metrics_long['id'].str[3:]\n",
    "metrics_long = metrics_long.set_index('id')\n",
    "metrics_long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "04ec565c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# snakemake.params.folder\n",
    "try:\n",
    "    models = snakemake.params.models  # snakefile would need to be\n",
    "except AttributeError:\n",
    "    models = ['Median', 'interpolated', 'CF', 'DAE', 'VAE']\n",
    "models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "919e5a74",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_by = ['data_split', 'data level', 'metric_name', 'model']\n",
    "\n",
    "selected_cols = ['metric_value', 'latent_dim', 'hidden_layers', 'n_params', 'text', 'N', 'M', 'id']\n",
    "\n",
    "order_categories = {'data level': ['proteinGroups', 'peptides', 'evidence'],\n",
    "                    'model': [*models]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "231f1cc9-ed4f-42d1-8d3c-b857917d6c97",
   "metadata": {},
   "outputs": [],
   "source": [
    "FOLDER = fname.parent.parent\n",
    "print(f\"{FOLDER =}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd460d5a-08b9-471a-8a72-97979a5a6338",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Annotation & Dump of metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "aa40341f-48c0-4c3f-a3b5-2991292f6e00",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_long.loc[metrics_long['model'].isin(['interpolated', 'median']), ['latent_dim', 'hidden_layers']] = '-'\n",
    "metrics_long['hidden_layers'] = metrics_long['hidden_layers'].fillna('-')\n",
    "metrics_long['text'] = 'LD: ' + metrics_long['latent_dim'].astype(str) + ' HL: ' + metrics_long['hidden_layers']\n",
    "\n",
    "# save metrics\n",
    "fname = 'metrics_long'\n",
    "metrics_long.to_csv(FOLDER / f'{fname}.csv')\n",
    "metrics_long.to_excel(FOLDER / f'{fname}.xlsx')\n",
    "\n",
    "metrics_long[['latent_dim', 'hidden_layers', 'model', 'text', ]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51c500cf-d958-41c2-8bd6-48a57015e878",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "accc970f-52f5-49ef-9067-3845b0c9d5ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "_unique = metrics_long[\"data level\"].unique()\n",
    "order_categories['data level'] = [l for l in order_categories['data level'] if l in _unique]  # ensure predefined order\n",
    "_unique = metrics_long['model'].unique()\n",
    "order_categories['model'] = [m for m in order_categories['model'] if m in _unique]  # ensure predefined order\n",
    "\n",
    "semi_supervised = [m for m in ['CF', 'DAE', 'VAE'] if m in _unique]\n",
    "reference = [m for m in ['median', 'interpolated'] if m in _unique]\n",
    "\n",
    "IDX_ORDER = (order_categories['data level'],\n",
    "             order_categories['model'])\n",
    "\n",
    "METRIC = 'MAE'\n",
    "\n",
    "order_categories"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51859bb1-ab7b-4a74-af3a-aecca7b3f285",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Select best models\n",
    "\n",
    "- based on validation data\n",
    "- report results on test data and validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0843c10e-21db-4e24-8769-181eb6c43fc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'valid_fake_na'  # data_split\n",
    "top_n = 3\n",
    "selected = (metrics_long\n",
    "            .reset_index()\n",
    "            .groupby(by=group_by)\n",
    "            .apply(\n",
    "                lambda df: df.sort_values(by='metric_value').iloc[:top_n]\n",
    "            )\n",
    "            )\n",
    "sel_on_val = selected.loc[\n",
    "    pd.IndexSlice[dataset, IDX_ORDER[0], 'MAE', IDX_ORDER[1]],\n",
    "    selected_cols]\n",
    "fname = FOLDER / f'sel_on_val_{dataset}_top_{top_n}.xlsx'\n",
    "writer = pd.ExcelWriter(fname)\n",
    "sel_on_val.to_excel(writer, sheet_name=f'top_{top_n}')\n",
    "sel_on_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dd7205c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select best model of top N with least parameters\n",
    "sel_on_val = (sel_on_val\n",
    "              .groupby(by=group_by)\n",
    "              .apply(\n",
    "                  lambda df: df.sort_values(by='n_params').iloc[0]\n",
    "              )\n",
    "              ).loc[\n",
    "    pd.IndexSlice[dataset, IDX_ORDER[0], 'MAE', IDX_ORDER[1]],\n",
    "    selected_cols]\n",
    "sel_on_val.to_excel(writer, sheet_name=f'selected')\n",
    "writer.close()\n",
    "sel_on_val"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c92b31dd-d024-47e2-b00d-c5880deb3c20",
   "metadata": {},
   "outputs": [],
   "source": [
    "Retrieve test data values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d02cf370-5ef9-4e65-ab54-c55b8cd58217",
   "metadata": {},
   "outputs": [],
   "source": [
    "sel_on_val = sel_on_val.set_index(['latent_dim', 'hidden_layers', 'id'], append=True)\n",
    "idx = sel_on_val.droplevel(level='data_split').index\n",
    "sel_on_val = sel_on_val.reset_index(['latent_dim', 'hidden_layers', 'id'])\n",
    "\n",
    "test_results = (metrics_long\n",
    "                .query('data_split == \"test_fake_na\"')\n",
    "                .reset_index().set_index(idx.names)\n",
    "                .loc[idx]\n",
    "                .reset_index(['latent_dim', 'hidden_layers', 'id'])\n",
    "                .set_index('data_split', append=True)\n",
    "                )[selected_cols]\n",
    "test_results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6835724e-7a17-4893-8724-a55b6de8d1a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "### test data results\n",
    "\n",
    "- selected on validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0fe1c0f7-6165-420a-96ca-912962cb0a0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_results = test_results.droplevel(['metric_name']).reset_index().set_index(['model', 'data level'])\n",
    "test_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "90223412-0a92-4e64-8358-9b72c4127c3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = test_results.loc[pd.IndexSlice[IDX_ORDER[1], IDX_ORDER[0]], 'text']\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "43fdcf27-2737-4782-b5cd-f2916b460766",
   "metadata": {},
   "outputs": [],
   "source": [
    "_to_plot = test_results['metric_value'].unstack(0).loc[IDX_ORDER]\n",
    "_to_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4d3d24fc-a8f6-4cd3-846b-2aaedf8fc85c",
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = 'best_models_1_test_mpl'\n",
    "_to_plot.to_excel(FOLDER / f'{fname}.xlsx')\n",
    "_to_plot.columns.name = ''\n",
    "ax = _to_plot.plot.bar(rot=0,\n",
    "                       xlabel='',\n",
    "                       ylabel=f\"{METRIC} (log2 intensities)\",\n",
    "                       width=.8)\n",
    "ax = pimmslearn.plotting.add_height_to_barplot(ax, size=12)\n",
    "ax = pimmslearn.plotting.add_text_to_barplot(ax, text, size=12)\n",
    "fig = ax.get_figure()\n",
    "fig.tight_layout()\n",
    "pimmslearn.savefig(fig, fname, folder=FOLDER)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39a0c7e8-9d0d-40fb-a9ad-b99917103f63",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Validation data results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "faf8f1a6-793b-473e-94ed-dde10b8dade9",
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = 'best_models_1_val_mpl'\n",
    "\n",
    "_to_plot = sel_on_val.reset_index(level=['data level', 'model']).loc[[('valid_fake_na', METRIC), ]]\n",
    "\n",
    "_to_plot = _to_plot.set_index(['data level', 'model'])[['metric_value', 'text']]\n",
    "_to_plot = _to_plot.loc[IDX_ORDER, :]\n",
    "_to_plot.index.name = ''\n",
    "# text = test_results['text'].unstack().loc[IDX_ORDER].unstack()\n",
    "_to_plot = _to_plot['metric_value'].unstack().loc[IDX_ORDER]\n",
    "_to_plot.to_csv(FOLDER / f'{fname}.csv')\n",
    "_to_plot.to_excel(FOLDER / f'{fname}.xlsx')\n",
    "# display(text.to_frame('text'))\n",
    "_to_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d27c680a-1634-4b07-9e5b-635320c107b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "_to_plot.columns.name = ''\n",
    "ax = _to_plot.plot.bar(rot=0,\n",
    "                       xlabel='',\n",
    "                       ylabel=f\"{METRIC} (log2 intensities)\",\n",
    "                       width=.8)\n",
    "ax = pimmslearn.plotting.add_height_to_barplot(ax, size=12)\n",
    "ax = pimmslearn.plotting.add_text_to_barplot(ax, text, size=12)\n",
    "fig = ax.get_figure()\n",
    "fig.tight_layout()\n",
    "pimmslearn.savefig(fig, fname, folder=FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e2fc9649-333e-4299-8e2e-6beb3e229589",
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = 'best_models_1_val_plotly'\n",
    "_to_plot = sel_on_val.reset_index(level=['data level', 'model']).loc[[('valid_fake_na', METRIC), ]]\n",
    "_to_plot = _to_plot.set_index(['data level', 'model'])\n",
    "_to_plot[['metric_value', 'latent_dim', 'hidden_layers', 'text']] = _to_plot[[\n",
    "    'metric_value', 'latent_dim', 'hidden_layers', 'text']].fillna('-')\n",
    "\n",
    "_to_plot = _to_plot.loc[pd.IndexSlice[IDX_ORDER], :]\n",
    "_to_plot.to_csv(FOLDER / f\"{fname}.csv\")\n",
    "_to_plot.to_excel(FOLDER / f\"{fname}.xlsx\")\n",
    "_to_plot[['metric_value', 'latent_dim', 'hidden_layers', 'text', 'N', 'n_params']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3b262a58-30b1-4cfe-b94f-69005d41cf40",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.bar(_to_plot.reset_index(),\n",
    "             x='data level',\n",
    "             y='metric_value',\n",
    "             hover_data={'N': ':,d', 'n_params': ':,d'},  # format hover data\n",
    "             color='model',\n",
    "             barmode=\"group\",\n",
    "             text='text',\n",
    "             labels={'metric_value': f\"{METRIC} (log2 intensities)\", 'data level': ''},\n",
    "             category_orders=order_categories,\n",
    "             template='none',\n",
    "             height=600)\n",
    "fig.update_layout(legend_title_text='')\n",
    "fig.write_html(FOLDER / f\"{fname}.html\")\n",
    "fig"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3db95765-f5fa-4f6c-86b5-afb8226f8cb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Order by best model setup over all datasets\n",
    "\n",
    "- select best average model on validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "eb0a0e33-4623-43c4-805d-ce11abec7957",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_by = ['model', 'latent_dim', 'hidden_layers']\n",
    "data_split = 'valid_fake_na'\n",
    "\n",
    "metrics_long_sel = metrics_long.query(f'data_split == \"{data_split}\"'\n",
    "                                      f' & metric_name == \"{METRIC}\"')\n",
    "\n",
    "best_on_average = metrics_long_sel.reset_index(\n",
    ").groupby(by=group_by\n",
    "          )['metric_value'].mean().sort_values().reset_index(level=group_by[1:])\n",
    "best_on_average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "bc504603-d4be-4279-bde3-54173ff56a9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_on_average.to_csv(FOLDER / 'average_performance_over_data_levels.csv')\n",
    "best_on_average = best_on_average.groupby(group_by[0]).apply(\n",
    "    lambda df: df.sort_values(by='metric_value').iloc[0]).set_index(group_by[1:], append=True)\n",
    "best_on_average"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9dbeb46-b7ac-4bf3-a3dc-ceff7c986709",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Test split results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "acf32e02-6717-405a-82d7-f8e70a7ad11b",
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = 'average_performance_over_data_levels_best_test'\n",
    "data_split = 'test_fake_na'\n",
    "\n",
    "metrics_long_sel_test = metrics_long.query(f'data_split == \"{data_split}\"'\n",
    "                                           f' & metric_name == \"{METRIC}\"')\n",
    "\n",
    "to_plot = (metrics_long_sel_test\n",
    "           .reset_index().set_index(group_by)\n",
    "           .loc[best_on_average.index]\n",
    "           .reset_index().set_index(['model', 'data level'])\n",
    "           .loc[pd.IndexSlice[order_categories['model'], order_categories['data level']], :])\n",
    "\n",
    "\n",
    "to_plot = to_plot.reset_index()\n",
    "to_plot['model annotated'] = to_plot['model'] + ' - ' + to_plot['text']\n",
    "order_model = to_plot['model annotated'].drop_duplicates().to_list()  # model name with annotation\n",
    "\n",
    "to_plot = to_plot.drop_duplicates(subset=['model', 'data level', 'metric_value'])\n",
    "to_plot.to_csv(FOLDER / f\"{fname}.csv\")\n",
    "to_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2f39a9a0-98ca-4d44-b900-62a8d6777db8",
   "metadata": {},
   "outputs": [],
   "source": [
    "figsize = (10, 8)  # None # (10,8)\n",
    "fig, ax = plt.subplots(figsize=figsize)\n",
    "to_plot.columns.name = ''\n",
    "ax = (to_plot\n",
    "      .set_index(['model annotated', 'data level'])['metric_value']\n",
    "      .unstack().rename_axis('', axis=1)\n",
    "      .loc[order_model, order_categories['data level']]\n",
    "      .plot.bar(\n",
    "          # xlabel=\"model with overall best performance for all datasets\",\n",
    "          xlabel='',\n",
    "          ylabel=\"MAE (log2 intensity)\",\n",
    "          rot=45,\n",
    "          width=.8,\n",
    "          ax=ax,\n",
    "          # colormap=\"Paired\",\n",
    "          color=[\n",
    "              '#a6cee3',\n",
    "              '#1f78b4',\n",
    "              '#b2df8a',\n",
    "              '#33a02c',\n",
    "              '#fb9a99',\n",
    "              '#e31a1c',\n",
    "              '#fdbf6f',\n",
    "              '#ff7f00',\n",
    "              '#cab2d6',\n",
    "              '#6a3d9a',\n",
    "              '#ffff99',\n",
    "              '#b15928']\n",
    "      )\n",
    "      )\n",
    "ax = pimmslearn.plotting.add_height_to_barplot(ax, size=11)\n",
    "ax.set_xticklabels(ax.get_xticklabels(), rotation=45, horizontalalignment='right')\n",
    "fig.tight_layout()\n",
    "pimmslearn.savefig(fig, fname, folder=FOLDER)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6348100a-10bf-49bc-9bbc-89a1654a111d",
   "metadata": {},
   "outputs": [],
   "source": [
    "plotly version with additional information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d31f39fb-a7a0-4573-8833-3c16efe8f17e",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.bar(to_plot,\n",
    "             x='model',\n",
    "             y='metric_value',\n",
    "             color='data level',\n",
    "             hover_data={'N': ':,d', 'n_params': ':,d'},  # format hover data\n",
    "             barmode=\"group\",\n",
    "             color_discrete_sequence=px.colors.colorbrewer.Paired,\n",
    "             # color_discrete_sequence=['#a6cee3', '#1f78b4', '#b2df8a'],\n",
    "             text='text',\n",
    "             labels={'metric_value': f\"{METRIC} (log2 intensities)\"},\n",
    "             category_orders=order_categories,\n",
    "             template='none',\n",
    "             height=600)\n",
    "fig.update_xaxes(title='')\n",
    "fig.write_image(FOLDER / f\"{fname}_plotly.pdf\")\n",
    "fig.update_layout(legend_title_text='')\n",
    "fig"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b96babfd-172c-4085-8133-7f74abd92841",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Validation data results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "cc9f7c57-5ce7-4a14-b518-ce532a667c4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = 'average_performance_over_data_levels_best_val'\n",
    "to_plot = (metrics_long_sel\n",
    "           .reset_index().set_index(group_by)\n",
    "           .loc[best_on_average.index].reset_index()\n",
    "           .set_index(['model', 'data level'])\n",
    "           .loc[pd.IndexSlice[order_categories['model'], order_categories['data level']], :]\n",
    "           )\n",
    "\n",
    "to_plot = to_plot.reset_index()\n",
    "to_plot['model annotated'] = to_plot['model'] + ' - ' + to_plot['text']\n",
    "order_model = to_plot['model annotated'].drop_duplicates().to_list()  # model name with annotation\n",
    "\n",
    "to_plot = to_plot.drop_duplicates(subset=['model', 'data level', 'metric_value'])\n",
    "to_plot.to_csv(FOLDER / f\"{fname}.csv\")\n",
    "to_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "f78f7362-222f-4489-9e99-60f89fe426ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "figsize = (10, 8)  # None # (10,8)\n",
    "fig, ax = plt.subplots(figsize=figsize)\n",
    "to_plot.columns.name = ''\n",
    "ax = (to_plot\n",
    "      .set_index(['model annotated', 'data level'])['metric_value']\n",
    "      .unstack().rename_axis('', axis=1)\n",
    "      .loc[order_model, order_categories['data level']]\n",
    "      .plot.bar(\n",
    "          # xlabel=\"model with overall best performance for all datasets\",\n",
    "          xlabel='',\n",
    "          ylabel=\"MAE (log2 intensity)\",\n",
    "          rot=45,\n",
    "          width=.8,\n",
    "          ax=ax,\n",
    "          # colormap=\"Paired\",\n",
    "          color=[\n",
    "              '#a6cee3',\n",
    "              '#1f78b4',\n",
    "              '#b2df8a',\n",
    "              '#33a02c',\n",
    "              '#fb9a99',\n",
    "              '#e31a1c',\n",
    "              '#fdbf6f',\n",
    "              '#ff7f00',\n",
    "              '#cab2d6',\n",
    "              '#6a3d9a',\n",
    "              '#ffff99',\n",
    "              '#b15928']\n",
    "      )\n",
    "      )\n",
    "ax = pimmslearn.plotting.add_height_to_barplot(ax, size=11)\n",
    "ax.set_xticklabels(ax.get_xticklabels(), rotation=45, horizontalalignment='right')\n",
    "fig.tight_layout()\n",
    "pimmslearn.savefig(fig, fname, folder=FOLDER)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56d626a8-3ae7-4506-b3ae-001a244ff8ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "plotly version with additional information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "51d1f4a9-54fc-470d-8511-3982235b5725",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.bar(to_plot,\n",
    "             x='model',\n",
    "             y='metric_value',\n",
    "             color='data level',\n",
    "             hover_data={'N': ':,d', 'n_params': ':,d'},  # format hover data\n",
    "             barmode=\"group\",\n",
    "             color_discrete_sequence=px.colors.colorbrewer.Paired,\n",
    "             # color_discrete_sequence=['#a6cee3', '#1f78b4', '#b2df8a'],\n",
    "             text='text',\n",
    "             labels={'metric_value': f\"{METRIC} (log2 intensities)\"},\n",
    "             category_orders=order_categories,\n",
    "             template='none',\n",
    "             height=600)\n",
    "fig.update_xaxes(title='')\n",
    "fig.write_image(FOLDER / f\"{fname}_plotly.pdf\")\n",
    "fig.update_layout(legend_title_text='')\n",
    "fig"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py:percent"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "vscode": {
   "interpreter": {
    "hash": "cf83e9cb890c7f96eb0ae04f39a82254555f56a1a0ed2f03b23a8b40fe6cd31c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}