models_inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2263b2d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3386f81b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import argparse\n",
    "from models.data_process import get_datatensor_partitions, prepare_nonproto_features, generate_partition_datatensor\n",
    "from models.dataset import ProtospacerDataset, ProtospacerExtendedDataset\n",
    "from src.utils import create_directory, one_hot_encode, get_device, ReaderWriter, print_eval_results\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e0d06fb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "cmd_opt = argparse.ArgumentParser(description='Argparser for data')\n",
    "cmd_opt.add_argument('-model_name',  type=str, help = 'name of the model')\n",
    "cmd_opt.add_argument('-exp_name',  type=str, help = 'name of the experiment')\n",
    "\n",
    "cmd_opt.add_argument('-data_dir',  type=str,default = './data/', help = 'directory of the data')\n",
    "cmd_opt.add_argument('-target_dir',  type=str, default='processed',  help = 'folder name to save the processed data')\n",
    "cmd_opt.add_argument('-working_dir',  type=str, default='./', help = 'the main working directory')\n",
    "cmd_opt.add_argument('-output_path', type=str, help='path to save the trained model')\n",
    "cmd_opt.add_argument('-model_path', type=str, help='path to trained model')\n",
    "cmd_opt.add_argument('-random_seed', type=int,default=42)\n",
    "cmd_opt.add_argument('-epoch_num', type=int, default =200, help='number of training epochs')\n",
    "args, _ = cmd_opt.parse_known_args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b4579eb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_ready(args, normalize_opt = 'max', train_size=0.9, fdtype=torch.float32):\n",
    "    ## prepare the data\n",
    "    data_dir = args.data_dir + args.target_dir\n",
    "    data_partitions = ReaderWriter.read_data(data_dir + '/data_partitions.pkl')\n",
    "    data = ReaderWriter.read_data(data_dir + '/list_of_x_f_y.pkl')\n",
    "    x_protospacer, x_extended_f,x_non_protos_f, y = data\n",
    "\n",
    "    if args.model_name in {'CNN', 'FFN'}:\n",
    "        ## onehot-encode the protospacer features\n",
    "        proc_x_protospacer = one_hot_encode(x_protospacer)\n",
    "        proc_x_protospacer = proc_x_protospacer.reshape(proc_x_protospacer.shape[0], -1)\n",
    "    elif args.model_name in {'Transformer', 'RNN'}:\n",
    "        proc_x_protospacer = x_protospacer \n",
    "        \n",
    "    x_non_protos_f_df, x_non_protos_f_norm = prepare_nonproto_features(x_non_protos_f, normalize_opt)\n",
    "    \n",
    "    if args.exp_name == 'protospacer_extended':\n",
    "        x_non_protos_features = x_non_protos_f_norm\n",
    "    elif args.exp_name == 'protospacer':\n",
    "        x_non_protos_features = None\n",
    "    dpartitions, datatensor_partitions = get_datatensor_partitions(data_partitions,\n",
    "                                                                   args.model_name,\n",
    "                                                                   proc_x_protospacer,\n",
    "                                                                   y,\n",
    "                                                                   x_non_protos_features,\n",
    "                                                                   fdtype=fdtype,\n",
    "                                                                   train_size=train_size,\n",
    "                                                                   random_state=args.random_seed)\n",
    "    return dpartitions, datatensor_partitions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "defd7da3",
   "metadata": {},
   "source": [
    "### Run trained models and evaluate performance on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0b0ac861",
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.trainval_workflow import run_inference\n",
    "from src.utils import compute_eval_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f5a9eb9e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- max normalization ---\n",
      "Running model: FFN, exp_name: protospacer, saved at ./output/FFN_v2/protospacer/train_val\n",
      "cpu\n",
      "test\n",
      "model_name: FFN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: FFN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: FFN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: FFN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: FFN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "===============\n",
      "run_name: run_0\n",
      "run_name: run_1\n",
      "run_name: run_2\n",
      "run_name: run_3\n",
      "run_name: run_4\n",
      "--- max normalization ---\n",
      "Running model: FFN, exp_name: protospacer_extended, saved at ./output/FFN_v2/protospacer_extended/train_val\n",
      "cpu\n",
      "test\n",
      "model_name: FFN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: FFN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: FFN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: FFN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: FFN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "===============\n",
      "run_name: run_0\n",
      "run_name: run_1\n",
      "run_name: run_2\n",
      "run_name: run_3\n",
      "run_name: run_4\n",
      "--- max normalization ---\n",
      "Running model: CNN, exp_name: protospacer, saved at ./output/CNN_v2/protospacer/train_val\n",
      "cpu\n",
      "test\n",
      "model_name: CNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: CNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: CNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: CNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: CNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "===============\n",
      "run_name: run_0\n",
      "run_name: run_1\n",
      "run_name: run_2\n",
      "run_name: run_3\n",
      "run_name: run_4\n",
      "--- max normalization ---\n",
      "Running model: CNN, exp_name: protospacer_extended, saved at ./output/CNN_v2/protospacer_extended/train_val\n",
      "cpu\n",
      "test\n",
      "model_name: CNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: CNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: CNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: CNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: CNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "===============\n",
      "run_name: run_0\n",
      "run_name: run_1\n",
      "run_name: run_2\n",
      "run_name: run_3\n",
      "run_name: run_4\n",
      "--- max normalization ---\n",
      "Running model: RNN, exp_name: protospacer, saved at ./output/RNN_v2/protospacer/train_val\n",
      "cpu\n",
      "test\n",
      "model_name: RNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: RNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: RNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: RNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: RNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "===============\n",
      "run_name: run_0\n",
      "run_name: run_1\n",
      "run_name: run_2\n",
      "run_name: run_3\n",
      "run_name: run_4\n",
      "--- max normalization ---\n",
      "Running model: RNN, exp_name: protospacer_extended, saved at ./output/RNN_v2/protospacer_extended/train_val\n",
      "cpu\n",
      "test\n",
      "model_name: RNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: RNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: RNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: RNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: RNN\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "===============\n",
      "run_name: run_0\n",
      "run_name: run_1\n",
      "run_name: run_2\n",
      "run_name: run_3\n",
      "run_name: run_4\n",
      "--- max normalization ---\n",
      "Running model: Transformer, exp_name: protospacer, saved at ./output/Transformer_v2/protospacer/train_val\n",
      "cpu\n",
      "test\n",
      "model_name: Transformer\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: Transformer\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: Transformer\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: Transformer\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: Transformer\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "===============\n",
      "run_name: run_0\n",
      "run_name: run_1\n",
      "run_name: run_2\n",
      "run_name: run_3\n",
      "run_name: run_4\n",
      "--- max normalization ---\n",
      "Running model: Transformer, exp_name: protospacer_extended, saved at ./output/Transformer_v2/protospacer_extended/train_val\n",
      "cpu\n",
      "test\n",
      "model_name: Transformer\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: Transformer\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: Transformer\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: Transformer\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "test\n",
      "model_name: Transformer\n",
      "xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "===============\n",
      "run_name: run_0\n",
      "run_name: run_1\n",
      "run_name: run_2\n",
      "run_name: run_3\n",
      "run_name: run_4\n"
     ]
    }
   ],
   "source": [
    "gpu_index = 0\n",
    "res_desc = {}\n",
    "version=2\n",
    "for model_name in ['FFN', 'CNN', 'RNN', 'Transformer']:\n",
    "    args.model_name =  model_name# {'RNN','CNN', 'Transformer'}\n",
    "    res_desc[model_name] = {}\n",
    "    for exp_name in ['protospacer', 'protospacer_extended']:\n",
    "        args.exp_name = exp_name\n",
    "        model_path = os.path.join(args.working_dir, \n",
    "                                  'output', \n",
    "                                  f'{model_name}_v{version}',\n",
    "                                  exp_name)\n",
    "        dpartitions, datatensor_partitions = get_data_ready(args, \n",
    "                                                            normalize_opt='max',\n",
    "                                                            train_size=0.9, \n",
    "                                                            fdtype=torch.float32)\n",
    "\n",
    "        train_val_path = os.path.join(model_path, 'train_val')\n",
    "        test_path = os.path.join(model_path, 'test')\n",
    "        \n",
    "        print(f'Running model: {model_name}, exp_name: {exp_name}, saved at {train_val_path}')\n",
    "        a, b = run_inference(datatensor_partitions, \n",
    "                             train_val_path, \n",
    "                             test_path, \n",
    "                             gpu_index, \n",
    "                             to_gpu=True)\n",
    "        print('='*15)\n",
    "        res_desc[model_name][exp_name] = compute_eval_results_df(test_path, len(dpartitions))        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d060ba52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_name: FFN, exp_name: protospacer\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>run_0</th>\n",
       "      <th>run_1</th>\n",
       "      <th>run_2</th>\n",
       "      <th>run_3</th>\n",
       "      <th>run_4</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>stddev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>spearman</th>\n",
       "      <td>0.709838</td>\n",
       "      <td>0.733197</td>\n",
       "      <td>0.719227</td>\n",
       "      <td>0.717508</td>\n",
       "      <td>0.699367</td>\n",
       "      <td>0.715827</td>\n",
       "      <td>0.717508</td>\n",
       "      <td>0.012479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pearson</th>\n",
       "      <td>0.768133</td>\n",
       "      <td>0.765097</td>\n",
       "      <td>0.745308</td>\n",
       "      <td>0.740168</td>\n",
       "      <td>0.735278</td>\n",
       "      <td>0.750797</td>\n",
       "      <td>0.745308</td>\n",
       "      <td>0.014907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MAE</th>\n",
       "      <td>4.321987</td>\n",
       "      <td>4.451395</td>\n",
       "      <td>4.273986</td>\n",
       "      <td>4.441211</td>\n",
       "      <td>4.413304</td>\n",
       "      <td>4.380377</td>\n",
       "      <td>4.413304</td>\n",
       "      <td>0.078353</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             run_0     run_1     run_2     run_3     run_4      mean  \\\n",
       "spearman  0.709838  0.733197  0.719227  0.717508  0.699367  0.715827   \n",
       "pearson   0.768133  0.765097  0.745308  0.740168  0.735278  0.750797   \n",
       "MAE       4.321987  4.451395  4.273986  4.441211  4.413304  4.380377   \n",
       "\n",
       "            median    stddev  \n",
       "spearman  0.717508  0.012479  \n",
       "pearson   0.745308  0.014907  \n",
       "MAE       4.413304  0.078353  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============\n",
      "model_name: FFN, exp_name: protospacer_extended\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>run_0</th>\n",
       "      <th>run_1</th>\n",
       "      <th>run_2</th>\n",
       "      <th>run_3</th>\n",
       "      <th>run_4</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>stddev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>spearman</th>\n",
       "      <td>0.737713</td>\n",
       "      <td>0.743325</td>\n",
       "      <td>0.712711</td>\n",
       "      <td>0.725644</td>\n",
       "      <td>0.713233</td>\n",
       "      <td>0.726525</td>\n",
       "      <td>0.725644</td>\n",
       "      <td>0.013926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pearson</th>\n",
       "      <td>0.784637</td>\n",
       "      <td>0.778563</td>\n",
       "      <td>0.745203</td>\n",
       "      <td>0.748264</td>\n",
       "      <td>0.735172</td>\n",
       "      <td>0.758368</td>\n",
       "      <td>0.748264</td>\n",
       "      <td>0.021860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MAE</th>\n",
       "      <td>4.126849</td>\n",
       "      <td>4.057595</td>\n",
       "      <td>4.278763</td>\n",
       "      <td>4.311938</td>\n",
       "      <td>4.421915</td>\n",
       "      <td>4.239412</td>\n",
       "      <td>4.278763</td>\n",
       "      <td>0.146496</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             run_0     run_1     run_2     run_3     run_4      mean  \\\n",
       "spearman  0.737713  0.743325  0.712711  0.725644  0.713233  0.726525   \n",
       "pearson   0.784637  0.778563  0.745203  0.748264  0.735172  0.758368   \n",
       "MAE       4.126849  4.057595  4.278763  4.311938  4.421915  4.239412   \n",
       "\n",
       "            median    stddev  \n",
       "spearman  0.725644  0.013926  \n",
       "pearson   0.748264  0.021860  \n",
       "MAE       4.278763  0.146496  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============\n",
      "model_name: CNN, exp_name: protospacer\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>run_0</th>\n",
       "      <th>run_1</th>\n",
       "      <th>run_2</th>\n",
       "      <th>run_3</th>\n",
       "      <th>run_4</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>stddev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>spearman</th>\n",
       "      <td>0.769019</td>\n",
       "      <td>0.789019</td>\n",
       "      <td>0.763319</td>\n",
       "      <td>0.771624</td>\n",
       "      <td>0.758249</td>\n",
       "      <td>0.770246</td>\n",
       "      <td>0.769019</td>\n",
       "      <td>0.011702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pearson</th>\n",
       "      <td>0.829258</td>\n",
       "      <td>0.811868</td>\n",
       "      <td>0.793431</td>\n",
       "      <td>0.804372</td>\n",
       "      <td>0.801098</td>\n",
       "      <td>0.808006</td>\n",
       "      <td>0.804372</td>\n",
       "      <td>0.013601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MAE</th>\n",
       "      <td>3.703795</td>\n",
       "      <td>3.740852</td>\n",
       "      <td>4.105558</td>\n",
       "      <td>3.915196</td>\n",
       "      <td>3.766388</td>\n",
       "      <td>3.846358</td>\n",
       "      <td>3.766388</td>\n",
       "      <td>0.165661</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             run_0     run_1     run_2     run_3     run_4      mean  \\\n",
       "spearman  0.769019  0.789019  0.763319  0.771624  0.758249  0.770246   \n",
       "pearson   0.829258  0.811868  0.793431  0.804372  0.801098  0.808006   \n",
       "MAE       3.703795  3.740852  4.105558  3.915196  3.766388  3.846358   \n",
       "\n",
       "            median    stddev  \n",
       "spearman  0.769019  0.011702  \n",
       "pearson   0.804372  0.013601  \n",
       "MAE       3.766388  0.165661  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============\n",
      "model_name: CNN, exp_name: protospacer_extended\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>run_0</th>\n",
       "      <th>run_1</th>\n",
       "      <th>run_2</th>\n",
       "      <th>run_3</th>\n",
       "      <th>run_4</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>stddev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>spearman</th>\n",
       "      <td>0.764276</td>\n",
       "      <td>0.783819</td>\n",
       "      <td>0.764906</td>\n",
       "      <td>0.771028</td>\n",
       "      <td>0.760843</td>\n",
       "      <td>0.768974</td>\n",
       "      <td>0.764906</td>\n",
       "      <td>0.009074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pearson</th>\n",
       "      <td>0.825335</td>\n",
       "      <td>0.818067</td>\n",
       "      <td>0.794789</td>\n",
       "      <td>0.800273</td>\n",
       "      <td>0.802215</td>\n",
       "      <td>0.808136</td>\n",
       "      <td>0.802215</td>\n",
       "      <td>0.012937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MAE</th>\n",
       "      <td>3.808375</td>\n",
       "      <td>3.752360</td>\n",
       "      <td>3.845842</td>\n",
       "      <td>3.931996</td>\n",
       "      <td>3.737353</td>\n",
       "      <td>3.815185</td>\n",
       "      <td>3.808375</td>\n",
       "      <td>0.078479</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             run_0     run_1     run_2     run_3     run_4      mean  \\\n",
       "spearman  0.764276  0.783819  0.764906  0.771028  0.760843  0.768974   \n",
       "pearson   0.825335  0.818067  0.794789  0.800273  0.802215  0.808136   \n",
       "MAE       3.808375  3.752360  3.845842  3.931996  3.737353  3.815185   \n",
       "\n",
       "            median    stddev  \n",
       "spearman  0.764906  0.009074  \n",
       "pearson   0.802215  0.012937  \n",
       "MAE       3.808375  0.078479  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============\n",
      "model_name: RNN, exp_name: protospacer\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>run_0</th>\n",
       "      <th>run_1</th>\n",
       "      <th>run_2</th>\n",
       "      <th>run_3</th>\n",
       "      <th>run_4</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>stddev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>spearman</th>\n",
       "      <td>0.765320</td>\n",
       "      <td>0.786523</td>\n",
       "      <td>0.768976</td>\n",
       "      <td>0.766275</td>\n",
       "      <td>0.758588</td>\n",
       "      <td>0.769136</td>\n",
       "      <td>0.766275</td>\n",
       "      <td>0.010444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pearson</th>\n",
       "      <td>0.810613</td>\n",
       "      <td>0.815035</td>\n",
       "      <td>0.795311</td>\n",
       "      <td>0.797432</td>\n",
       "      <td>0.796956</td>\n",
       "      <td>0.803069</td>\n",
       "      <td>0.797432</td>\n",
       "      <td>0.009075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MAE</th>\n",
       "      <td>3.832806</td>\n",
       "      <td>3.877604</td>\n",
       "      <td>3.869510</td>\n",
       "      <td>3.945184</td>\n",
       "      <td>3.703842</td>\n",
       "      <td>3.845789</td>\n",
       "      <td>3.869510</td>\n",
       "      <td>0.089123</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             run_0     run_1     run_2     run_3     run_4      mean  \\\n",
       "spearman  0.765320  0.786523  0.768976  0.766275  0.758588  0.769136   \n",
       "pearson   0.810613  0.815035  0.795311  0.797432  0.796956  0.803069   \n",
       "MAE       3.832806  3.877604  3.869510  3.945184  3.703842  3.845789   \n",
       "\n",
       "            median    stddev  \n",
       "spearman  0.766275  0.010444  \n",
       "pearson   0.797432  0.009075  \n",
       "MAE       3.869510  0.089123  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============\n",
      "model_name: RNN, exp_name: protospacer_extended\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>run_0</th>\n",
       "      <th>run_1</th>\n",
       "      <th>run_2</th>\n",
       "      <th>run_3</th>\n",
       "      <th>run_4</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>stddev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>spearman</th>\n",
       "      <td>0.769832</td>\n",
       "      <td>0.778847</td>\n",
       "      <td>0.753758</td>\n",
       "      <td>0.750929</td>\n",
       "      <td>0.768922</td>\n",
       "      <td>0.764458</td>\n",
       "      <td>0.768922</td>\n",
       "      <td>0.011762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pearson</th>\n",
       "      <td>0.815824</td>\n",
       "      <td>0.803886</td>\n",
       "      <td>0.780654</td>\n",
       "      <td>0.756225</td>\n",
       "      <td>0.809744</td>\n",
       "      <td>0.793267</td>\n",
       "      <td>0.803886</td>\n",
       "      <td>0.024618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MAE</th>\n",
       "      <td>3.782751</td>\n",
       "      <td>3.880668</td>\n",
       "      <td>3.981464</td>\n",
       "      <td>4.264782</td>\n",
       "      <td>3.534493</td>\n",
       "      <td>3.888832</td>\n",
       "      <td>3.880668</td>\n",
       "      <td>0.267766</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             run_0     run_1     run_2     run_3     run_4      mean  \\\n",
       "spearman  0.769832  0.778847  0.753758  0.750929  0.768922  0.764458   \n",
       "pearson   0.815824  0.803886  0.780654  0.756225  0.809744  0.793267   \n",
       "MAE       3.782751  3.880668  3.981464  4.264782  3.534493  3.888832   \n",
       "\n",
       "            median    stddev  \n",
       "spearman  0.768922  0.011762  \n",
       "pearson   0.803886  0.024618  \n",
       "MAE       3.880668  0.267766  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============\n",
      "model_name: Transformer, exp_name: protospacer\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>run_0</th>\n",
       "      <th>run_1</th>\n",
       "      <th>run_2</th>\n",
       "      <th>run_3</th>\n",
       "      <th>run_4</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>stddev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>spearman</th>\n",
       "      <td>0.682484</td>\n",
       "      <td>0.703473</td>\n",
       "      <td>0.695263</td>\n",
       "      <td>0.724780</td>\n",
       "      <td>0.690616</td>\n",
       "      <td>0.699323</td>\n",
       "      <td>0.695263</td>\n",
       "      <td>0.016133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pearson</th>\n",
       "      <td>0.748878</td>\n",
       "      <td>0.755179</td>\n",
       "      <td>0.741985</td>\n",
       "      <td>0.745598</td>\n",
       "      <td>0.735357</td>\n",
       "      <td>0.745399</td>\n",
       "      <td>0.745598</td>\n",
       "      <td>0.007421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MAE</th>\n",
       "      <td>4.614115</td>\n",
       "      <td>4.626415</td>\n",
       "      <td>4.349097</td>\n",
       "      <td>4.427824</td>\n",
       "      <td>4.253950</td>\n",
       "      <td>4.454280</td>\n",
       "      <td>4.427824</td>\n",
       "      <td>0.163610</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             run_0     run_1     run_2     run_3     run_4      mean  \\\n",
       "spearman  0.682484  0.703473  0.695263  0.724780  0.690616  0.699323   \n",
       "pearson   0.748878  0.755179  0.741985  0.745598  0.735357  0.745399   \n",
       "MAE       4.614115  4.626415  4.349097  4.427824  4.253950  4.454280   \n",
       "\n",
       "            median    stddev  \n",
       "spearman  0.695263  0.016133  \n",
       "pearson   0.745598  0.007421  \n",
       "MAE       4.427824  0.163610  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============\n",
      "model_name: Transformer, exp_name: protospacer_extended\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>run_0</th>\n",
       "      <th>run_1</th>\n",
       "      <th>run_2</th>\n",
       "      <th>run_3</th>\n",
       "      <th>run_4</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>stddev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>spearman</th>\n",
       "      <td>0.719099</td>\n",
       "      <td>0.728455</td>\n",
       "      <td>0.721773</td>\n",
       "      <td>0.708179</td>\n",
       "      <td>0.706992</td>\n",
       "      <td>0.716899</td>\n",
       "      <td>0.719099</td>\n",
       "      <td>0.009170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pearson</th>\n",
       "      <td>0.779659</td>\n",
       "      <td>0.761188</td>\n",
       "      <td>0.765637</td>\n",
       "      <td>0.732273</td>\n",
       "      <td>0.750374</td>\n",
       "      <td>0.757826</td>\n",
       "      <td>0.761188</td>\n",
       "      <td>0.017731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MAE</th>\n",
       "      <td>4.266432</td>\n",
       "      <td>4.219082</td>\n",
       "      <td>4.139365</td>\n",
       "      <td>4.481317</td>\n",
       "      <td>4.076254</td>\n",
       "      <td>4.236490</td>\n",
       "      <td>4.219082</td>\n",
       "      <td>0.155120</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             run_0     run_1     run_2     run_3     run_4      mean  \\\n",
       "spearman  0.719099  0.728455  0.721773  0.708179  0.706992  0.716899   \n",
       "pearson   0.779659  0.761188  0.765637  0.732273  0.750374  0.757826   \n",
       "MAE       4.266432  4.219082  4.139365  4.481317  4.076254  4.236490   \n",
       "\n",
       "            median    stddev  \n",
       "spearman  0.719099  0.009170  \n",
       "pearson   0.761188  0.017731  \n",
       "MAE       4.219082  0.155120  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============\n"
     ]
    }
   ],
   "source": [
    "for model_name in res_desc:\n",
    "    for exp_name in res_desc[model_name]:\n",
    "        print(f'model_name: {model_name}, exp_name: {exp_name}')\n",
    "        display(res_desc[model_name][exp_name])\n",
    "        print('='*15)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc346ac7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5b1e214",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}