notebook_templates/Table 4 p_values.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "from concurrent.futures import ProcessPoolExecutor\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "from omegaconf import OmegaConf\n",
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sns\n",
    "from matplotlib.gridspec import GridSpec\n",
    "from tqdm.notebook import tqdm\n",
    "import bm.train\n",
    "import os\n",
    "\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "os.chdir(Path(bm.train.__file__).parent.parent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "output_dir = bm.train.main.dora.dir\n",
    "eval_dir = output_dir / \"eval\" / \"signatures\"\n",
    "sigs_to_eval = [p.name for p in (output_dir / \"grids\" / \"ablation_final\").iterdir()]\n",
    "assert output_dir.exists()\n",
    "assert eval_dir.exists()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set()\n"
     ]
    }
   ],
   "source": [
    "# Select signatures\n",
    "valid_sigs = [sig for sig in sigs_to_eval if (eval_dir / sig / \"vocab_segment.pth\").is_file()]\n",
    "print(set(sigs_to_eval) - set(valid_sigs))\n",
    "configs = [OmegaConf.load(eval_dir / sig / \"solver_config.yaml\") for sig in valid_sigs]\n",
    "run_df = pd.DataFrame({\n",
    "    \"sig\":valid_sigs,\n",
    "})\n",
    "run_df[\"dataset\"] = [\"-\".join(conf.dset.selections) for conf in configs]\n",
    "run_df[\"seed\"] = [conf.seed for conf in configs]\n",
    "run_df[\"features\"] = [\"-\".join(conf.dset.features) for conf in configs]\n",
    "run_df[\"loss\"] = [conf.optim.loss for conf in configs]\n",
    "# run_df[\"is_random\"] = [conf.test.wer_random for conf in configs]\n",
    "run_df[\"max_scale\"] = [conf.norm.max_scale for conf in configs]\n",
    "run_df[\"n_mels\"] = [conf.dset.features_params.MelSpectrum.n_mels for conf in configs]\n",
    "run_df[\"deepmel\"] = [bool(conf.clip.arch) for conf in configs]\n",
    "run_df[\"ft\"] = [conf.optim.epochs == 1 and not conf.test.wer_random for conf in configs]\n",
    "run_df[\"random\"] = [conf.test.wer_random for conf in configs]\n",
    "\n",
    "run_df[\"batch_size\"] = [conf.optim.batch_size for conf in configs]\n",
    "run_df[\"ft\"] = [conf.optim.lr == 0 for conf in configs]\n",
    "run_df[\"dropout\"] = [conf.simpleconv.merger_dropout > 0 for conf in configs]\n",
    "run_df[\"gelu\"] = [bool(conf.simpleconv.gelu) for conf in configs]\n",
    "run_df[\"skip\"] = [bool(conf.simpleconv.skip) for conf in configs]\n",
    "run_df[\"initial\"] = [bool(conf.simpleconv.initial_linear) for conf in configs]\n",
    "run_df[\"complex\"] = [bool(conf.simpleconv.complex_out) for conf in configs]\n",
    "run_df[\"subject_lay\"] = [bool(conf.simpleconv.subject_layers) for conf in configs]\n",
    "run_df[\"subject_emb\"] = [bool(conf.simpleconv.subject_dim) for conf in configs]\n",
    "run_df[\"attention\"] = [bool(conf.simpleconv.merger) for conf in configs]\n",
    "run_df[\"glu\"] = [bool(conf.simpleconv.glu) for conf in configs]\n",
    "run_df[\"depth\"] = [conf.simpleconv.depth for conf in configs]\n",
    "\n",
    "def get_name(row):\n",
    "    if not row.dropout:\n",
    "        return r\"\\wo spatial attention dropout\"\n",
    "    if not row.gelu:\n",
    "        return r\"\\wo GELU, \\w ReLU\"\n",
    "    if not row.skip:\n",
    "        return r\"\\wo skip connections\"\n",
    "    if not row.initial:\n",
    "        return r\"\\wo initial 1x1 conv.\"\n",
    "    if not row.complex:\n",
    "        return r\"\\wo final convs\"\n",
    "    if not row.attention:\n",
    "        return r\"\\wo spatial attention\"\n",
    "    if not row.glu:\n",
    "        return r\"\\wo non-residual GLU conv.\"\n",
    "    if not row.subject_lay:\n",
    "        if row.subject_emb:\n",
    "            return r\"\\w subj. embedding*\"\n",
    "        else:\n",
    "            return r\"\\wo subject-specific layer\"\n",
    "    if row.depth == 5:\n",
    "        return r\"less deep\"\n",
    "    elif row.max_scale != 20:\n",
    "        if row.max_scale == 100:\n",
    "            return \"\\w clamp=100\"\n",
    "        else:\n",
    "            if row.autoreject:\n",
    "                return \"autoreject\"\n",
    "            return \"\\wo clamping brain signal\"\n",
    "    return \"Our model\"\n",
    "    \n",
    "run_df['name'] = run_df.apply(get_name, axis=1)\n",
    "run_df = run_df[(run_df.loss == \"clip\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our model 12\n",
      "\\wo spatial attention 12\n",
      "\\wo spatial attention dropout 12\n",
      "\\wo non-residual GLU conv. 12\n",
      "\\wo initial 1x1 conv. 12\n",
      "\\wo GELU, \\w ReLU 12\n",
      "\\wo skip connections 12\n",
      "\\wo final convs 12\n",
      "\\wo subject-specific layer 12\n",
      "\\w subj. embedding* 12\n",
      "\\wo clamping brain signal 12\n"
     ]
    }
   ],
   "source": [
    "for name in run_df.name.unique():\n",
    "    print(name, (run_df.name == name).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['audio_mous',\n",
       " 'audio_mous',\n",
       " 'audio_mous',\n",
       " 'brennan2019',\n",
       " 'brennan2019',\n",
       " 'brennan2019',\n",
       " 'broderick2019',\n",
       " 'broderick2019',\n",
       " 'broderick2019',\n",
       " 'gwilliams2022',\n",
       " 'gwilliams2022',\n",
       " 'gwilliams2022']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_df.sort_values('dataset', ignore_index=True, inplace=True)\n",
    "run_df[run_df.name == 'base'].dataset.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['audio_mous', 'audio_mous', 'audio_mous', 'brennan2019', 'brennan2019', 'brennan2019', 'broderick2019', 'broderick2019', 'broderick2019', 'gwilliams2022', 'gwilliams2022', 'gwilliams2022']\n",
      "[2036, 2037, 2038, 2036, 2037, 2038, 2036, 2037, 2038, 2036, 2037, 2038]\n"
     ]
    }
   ],
   "source": [
    "# For main table \n",
    "# %time\n",
    "run_df.sort_values(['dataset', 'seed'], ignore_index=True, inplace=True)\n",
    "table_sigs = {}\n",
    "table_sigs['reference'] = run_df[run_df.name == \"base\"].sig.tolist()\n",
    "for name in run_df.name.unique():\n",
    "    if name == \"base\": continue\n",
    "    table_sigs[name] = run_df[run_df.name == name].sig.tolist()\n",
    "table_dataset_names = run_df[run_df.name == 'base'].dataset.tolist()\n",
    "seed_names = run_df[run_df.name == 'base'].seed.tolist()\n",
    "print(table_dataset_names)\n",
    "print(seed_names)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Common functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_data_from_sig(sig, level=\"word\"):\n",
    "    \"\"\"\n",
    "    Load data from solver signature\n",
    "    - probs (torch tensor): probability on vocab [N, V]\n",
    "    - vocab (torch tensor): vocab of word hashes [V]\n",
    "    - words (torch tensor): the word hash for each sample [N]\n",
    "    - metadata (panda dataframe) of len [N] which contains for each sample:\n",
    "           'word_hashes', 'word_indices', 'seq_indices',\n",
    "           'word_strings', 'subject_id', 'recording_id'\n",
    "    \"\"\"\n",
    "    assert level in [\"word\", \"segment\"], \"level should be 'word' or 'segment'\"\n",
    "    probs = torch.load(eval_dir / sig / f\"probs_{level}.pth\") \n",
    "    vocab = torch.load(eval_dir / sig / f\"vocab_{level}.pth\") # vocab (hashes)\n",
    "    metadata = pd.read_csv(eval_dir / sig / \"metadata.csv\", index_col=0) \n",
    "    words = torch.LongTensor(metadata[f\"{level}_hashes\"]) # for each sample, the word (hashes)\n",
    "    assert probs.shape == (len(words), len(vocab))\n",
    "    assert len(words) == len(metadata)\n",
    "    metadata[\"idx\"] = range(len(words))\n",
    "    metadata[\"word_strings\"] = metadata[\"word_strings\"].str.lower()\n",
    "\n",
    "    return probs, vocab, words, metadata\n",
    "\n",
    "\n",
    "def get_accuracy_from_probs(probs, row_labels, col_labels, topk=10):\n",
    "    \"\"\"\n",
    "    probs: for each row, the probability distribution over a vocab\n",
    "    returns the topk accuracy that the topk best predicted labels\n",
    "    match the row_labels\n",
    "    Inputs:\n",
    "        probs: of shape [B, V] probability over vocab, each row sums to 1\n",
    "        row_labels: of shape [B] true word for each row\n",
    "        col_labels: [V] word that correspond to each column\n",
    "        topk: int\n",
    "    Returns: float scalar, topk accuracy\n",
    "    \"\"\"\n",
    "    assert len(row_labels) == len(probs)\n",
    "    assert len(col_labels) == probs.shape[1]\n",
    "\n",
    "    # Extract topk indices\n",
    "    idx = probs.topk(topk, dim=1).indices\n",
    "\n",
    "    # Get the corresponding topk labels\n",
    "    whs = col_labels[idx.view(-1)].reshape(idx.shape)\n",
    "\n",
    "    # 1 if the labels matches with the targets\n",
    "    correct = ((whs == row_labels[:, None]).any(1)).float()\n",
    "    assert len(correct) == len(row_labels)\n",
    "\n",
    "    # Average across samples\n",
    "    acc = correct.mean()\n",
    "\n",
    "    return acc.item()\n",
    "\n",
    "def eval_acc(sigs, level=\"word\", add_baselines=True, per_sub=False):\n",
    "    \"\"\"\n",
    "    Return accuracy dataframe for multiple sigs \n",
    "    level: whether to return word or segment level accuracy\n",
    "    \"\"\"\n",
    "    futures = []\n",
    "    acc = []\n",
    "    with ProcessPoolExecutor(20) as pool:\n",
    "        for sig in sigs:\n",
    "            future = pool.submit(eval_acc_one_sig, sig, level=level, add_baselines=add_baselines, per_sub=per_sub)\n",
    "            futures.append((sig, future))\n",
    "        for sig, future in tqdm(futures):\n",
    "            try:\n",
    "                acc_sig = future.result()\n",
    "            except Exception:\n",
    "                print(\"ERROR WITH\", sig)\n",
    "                continue\n",
    "            acc_sig[\"sig\"] = sig\n",
    "            acc.append(acc_sig)\n",
    "    acc = pd.concat(acc)\n",
    "    return acc\n",
    "\n",
    "\n",
    "def eval_acc_nopool(sigs, level=\"word\", add_baselines=True, per_sub=False):\n",
    "    \"\"\"\n",
    "    Return accuracy dataframe for multiple sigs \n",
    "    level: whether to return word or segment level accuracy\n",
    "    \"\"\"\n",
    "    acc = []\n",
    "    for sig in tqdm(sigs):\n",
    "        try:\n",
    "            acc_sig = eval_acc_one_sig(sig, level=level, add_baselines=add_baselines, per_sub=per_sub)\n",
    "        except Exception:\n",
    "            print(\"ERROR WITH\", sig)\n",
    "            continue\n",
    "        acc_sig[\"sig\"] = sig\n",
    "        acc.append(acc_sig)\n",
    "    acc = pd.concat(acc)\n",
    "    return acc\n",
    "\n",
    "def eval_acc_one_sig(sig, topks=(1, 5, 10), level=\"word\", add_baselines=True, per_sub=False):\n",
    "    \"\"\"\n",
    "    Return accuracy dataframe from one solver signature\n",
    "    level: whether to return `word` or `segment` level accuracy\n",
    "    \"\"\"\n",
    "    # Load data\n",
    "    probs, vocab, words, meta = load_data_from_sig(sig, level=level)\n",
    "    meta = meta.reset_index()\n",
    "    \n",
    "    if not per_sub:\n",
    "        meta[\"subject_id\"] = \"nan\"\n",
    "\n",
    "    # Compute acc\n",
    "    acc_df = []\n",
    "    for topk in topks:\n",
    "        for subject, metasub in meta.groupby(\"subject_id\"):\n",
    "            idx = metasub[\"index\"].values\n",
    "\n",
    "            # --- Acc ---\n",
    "            acc = get_accuracy_from_probs(probs[idx], words[idx], vocab, topk=topk)\n",
    "\n",
    "            out = {\n",
    "                \"acc\":acc,\n",
    "                \"topk\":topk,\n",
    "                \"subject_id\": subject, \n",
    "            }\n",
    "            if add_baselines:\n",
    "                # --- Baseline on vocab ---\n",
    "                # equivalent to : shuffle targets vocab (inf times)\n",
    "                # equivalent to : output uniform prob on vocab\n",
    "                # equivalent to : 1/vocab_len\n",
    "                rand_probs_vocab = torch.ones_like(probs[idx]) / len(vocab)\n",
    "                out[\"baseline_vocab\"] = get_accuracy_from_probs(rand_probs_vocab, words[idx], vocab, topk=topk)\n",
    "\n",
    "                # --- Baseline on words ---\n",
    "                # equivalent to : shuffle word targets before aggregating on vocab (inf times)\n",
    "                # equivalent to : output uniform prob on samples\n",
    "                # equivalent to : each_word_freq\n",
    "                check_vocab, counts = torch.unique(words, return_counts=True)\n",
    "                rand_probs_words = torch.stack([counts/len(words)]*len(probs[idx]))\n",
    "                out[\"baseline\"] = get_accuracy_from_probs(rand_probs_words, words[idx], vocab, topk=topk)\n",
    "                \n",
    "            # Update\n",
    "            acc_df.append(out)\n",
    "    acc_df = pd.DataFrame(acc_df)\n",
    "    return acc_df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for reference ['01cd28fc', '49efae33', '6b9ba19e', 'd605ce2f', 'bd231c7a', 'ce18f336', '1dd888f0', 'bd30ebf1', 'bd04f96e', 'ceb23552', 'bd7d06d2', 'e059e7e5']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d67ec708ab684bcd899a9bb579bedbcf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for \\wo clamping brain signal ['b476448e', 'bea7f00a', 'd1d37637', 'f3cae34e', '84f95be4', '586f1ad0', '22fb342c', '7f15adbc', 'c1150d28', 'b978fc86', '9ed67c53', 'fbdd79aa']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2af0dd3dd61144d2875e02b4e3ee4205",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for \\wo final convs ['480110a3', 'e521c72b', 'cfe0aefd', 'de507ded', '5f8b5d6a', 'bba535c9', '666e391f', 'a5e3ebae', '626f961e', '092d508e', '5b38fa38', '9567ff2c']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a382c0190c444bcb7b8aabd54575bdf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for \\wo skip connections ['fb2cced8', 'c4098073', 'ac741d87', '27332012', 'a781c6a7', '5349feef', 'bb4f4ee9', 'd3399072', '15546f79', '70a0e185', 'd9cec30d', '9783831e']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad28f5804c37499fa012c5d51146aaa1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for \\wo subject-specific layer ['9e8b929d', 'ac5b3b57', '91c5c41b', '2325aad9', '00e0bdcd', '71dd6af4', 'e48c36ea', '5c633e70', '76b0e85a', 'a5acd4a3', '4479100b', '78d5d473']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "135043b703da470eb126696a8e4ba2b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for \\wo initial 1x1 conv. ['8e9c33b1', '036741ea', '1481dbbb', '5d3de32c', '1d014dcd', '6019c47b', '08cf8898', '22d2260a', '87c1ffa6', '81bd9784', '0d4f43b4', '5d7a51d4']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7cf4de91fb304f04b1bbf359a83b6f9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for \\wo non-residual GLU conv. ['55751f43', '832d9622', '3a9ae364', '493023a0', '0f34bbaf', 'a4b730f1', 'fa2d2578', '2ae3913a', '850b80f6', 'ff85bfd6', '0896a4a8', '0f17594c']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d12abca37d5543d2b3ddc25b146ff447",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for \\w subj. embedding* ['f33cf48b', '3cf51478', '745a6231', '64c1362f', '8ababc44', 'b8d46745', '6577f78f', '0e6262e9', '203c64a9', 'c3b555db', '73c5dce9', 'd733b028']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb1d408dfe4a40a4bbde3c03b107bda2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for \\wo spatial attention ['ec29e81e', 'c32f25b6', '8a7419d6', '2aecc316', '437a096c', '58a6fa89', 'fc475d47', 'ea519a2a', 'd4cf1bbe', 'ca8c38f8', '1bd6de99', '57a34964']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "569288af889e4bcc8ba0081dba9fb7bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for \\wo GELU, \\w ReLU ['4438d221', '766c6d08', '4e9d4727', '16d17b9c', '957fe7a4', '0c80dd82', '38449131', '6324264b', '3d61e840', '048edc79', '86811260', '46c3d964']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f17cea81f1b84923a314f04119a2a68d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing acc for \\wo spatial attention dropout ['2dd6ddbe', 'b23c2cfa', '22ca6dba', 'd890f8eb', 'e5c6b552', '4c3a83bf', '41831434', '8d13203d', '5c64a266', '85f40582', '55c4a03c', 'd43b1b05']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29c9447abd704df290635bc36359550a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc_df = []\n",
    "for row_label, sigs in table_sigs.items():\n",
    "    print(f\"Computing acc for {row_label} {sigs}\")\n",
    "    row_acc = eval_acc_nopool(sigs, level=\"segment\", add_baselines=False, per_sub=True,)\n",
    "    row_acc[\"dataset\"] = [table_dataset_names[sigs.index(k)] for k in row_acc.sig.values]\n",
    "    row_acc[\"seed\"] = [seed_names[sigs.index(k)] for k in row_acc.sig.values]\n",
    "    row_acc[\"row_label\"] = row_label\n",
    "    acc_df.append(row_acc)\n",
    "acc_df = pd.concat(acc_df)\n",
    "\n",
    "def dset_order(name):\n",
    "    return name.map({\n",
    "       'audio_mous': 0, \n",
    "       'gwilliams2022': 1,\n",
    "       'broderick2019': 2,\n",
    "       'brennan2019': 3,\n",
    "    })\n",
    "\n",
    "acc_df = acc_df.sort_values([\"dataset\"], key=dset_order);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/private/home/defossez/miniconda3/envs/bm/lib/python3.8/site-packages/scipy/stats/_morestats.py:3414: UserWarning: Exact p-value calculation does not work if there are zeros. Switching to normal approximation.\n",
      "  warnings.warn(\"Exact p-value calculation does not work if there are \"\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import wilcoxon\n",
    "acc_df = acc_df[acc_df.topk==10]\n",
    "pivot = pd.pivot_table(acc_df, index=[\"dataset\", \"subject_id\"], columns=[\"row_label\"], values=\"acc\")\n",
    "pivot = pivot[table_sigs.keys()] # Reorder rows\n",
    "\n",
    "# Average score\n",
    "means = pivot.groupby(\"dataset\").agg(\"mean\").T\n",
    "\n",
    "# Standard Error of the Means (std too high)\n",
    "sems = pivot.groupby(\"dataset\").agg(\"sem\").T\n",
    "\n",
    "# Difference in score between reference and each row\n",
    "deltas = means - means.loc[\"reference\"]\n",
    "\n",
    "# Pvalues: grouped by dataset => 1 pvalue per row and dataset\n",
    "pvalues = pivot.groupby(\"dataset\").corr(method=lambda x, y: wilcoxon(x, y)[1])\n",
    "pvalues = pvalues.reset_index().query(\"row_label=='reference'\").set_index(\"dataset\").T.drop(\"row_label\")\n",
    "pvalues = pvalues.astype(float)\n",
    "\n",
    "# Pvalues: aggregated on datasets => 1 pvalue per row\n",
    "pvalues_agg_dset = pivot.corr(method=lambda x, y: wilcoxon(x, y)[1])[[\"reference\",]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>dataset</th>\n",
       "      <th>audio_mous</th>\n",
       "      <th>brennan2019</th>\n",
       "      <th>broderick2019</th>\n",
       "      <th>gwilliams2022</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>reference</th>\n",
       "      <td>0.675469</td>\n",
       "      <td>0.257410</td>\n",
       "      <td>0.134172</td>\n",
       "      <td>0.669869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo clamping brain signal</th>\n",
       "      <td>0.014541</td>\n",
       "      <td>0.140769</td>\n",
       "      <td>0.009778</td>\n",
       "      <td>0.226597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo final convs</th>\n",
       "      <td>0.674825</td>\n",
       "      <td>0.190210</td>\n",
       "      <td>0.111579</td>\n",
       "      <td>0.653442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo skip connections</th>\n",
       "      <td>0.654489</td>\n",
       "      <td>0.242466</td>\n",
       "      <td>0.109510</td>\n",
       "      <td>0.627220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo subject-specific layer</th>\n",
       "      <td>0.424489</td>\n",
       "      <td>0.201883</td>\n",
       "      <td>0.068915</td>\n",
       "      <td>0.447212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo initial 1x1 conv.</th>\n",
       "      <td>0.628566</td>\n",
       "      <td>0.220709</td>\n",
       "      <td>0.118565</td>\n",
       "      <td>0.641835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo non-residual GLU conv.</th>\n",
       "      <td>0.669520</td>\n",
       "      <td>0.060001</td>\n",
       "      <td>0.068657</td>\n",
       "      <td>0.664579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\w subj. embedding*</th>\n",
       "      <td>0.647897</td>\n",
       "      <td>0.293972</td>\n",
       "      <td>0.146858</td>\n",
       "      <td>0.670004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo spatial attention</th>\n",
       "      <td>0.658781</td>\n",
       "      <td>0.205716</td>\n",
       "      <td>0.118991</td>\n",
       "      <td>0.621047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo GELU, \\w ReLU</th>\n",
       "      <td>0.658231</td>\n",
       "      <td>0.245611</td>\n",
       "      <td>0.125274</td>\n",
       "      <td>0.652161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo spatial attention dropout</th>\n",
       "      <td>0.674823</td>\n",
       "      <td>0.268156</td>\n",
       "      <td>0.123240</td>\n",
       "      <td>0.654153</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "dataset                        audio_mous  brennan2019  broderick2019  \\\n",
       "row_label                                                               \n",
       "reference                        0.675469     0.257410       0.134172   \n",
       "\\wo clamping brain signal        0.014541     0.140769       0.009778   \n",
       "\\wo final convs                  0.674825     0.190210       0.111579   \n",
       "\\wo skip connections             0.654489     0.242466       0.109510   \n",
       "\\wo subject-specific layer       0.424489     0.201883       0.068915   \n",
       "\\wo initial 1x1 conv.            0.628566     0.220709       0.118565   \n",
       "\\wo non-residual GLU conv.       0.669520     0.060001       0.068657   \n",
       "\\w subj. embedding*              0.647897     0.293972       0.146858   \n",
       "\\wo spatial attention            0.658781     0.205716       0.118991   \n",
       "\\wo GELU, \\w ReLU                0.658231     0.245611       0.125274   \n",
       "\\wo spatial attention dropout    0.674823     0.268156       0.123240   \n",
       "\n",
       "dataset                        gwilliams2022  \n",
       "row_label                                     \n",
       "reference                           0.669869  \n",
       "\\wo clamping brain signal           0.226597  \n",
       "\\wo final convs                     0.653442  \n",
       "\\wo skip connections                0.627220  \n",
       "\\wo subject-specific layer          0.447212  \n",
       "\\wo initial 1x1 conv.               0.641835  \n",
       "\\wo non-residual GLU conv.          0.664579  \n",
       "\\w subj. embedding*                 0.670004  \n",
       "\\wo spatial attention               0.621047  \n",
       "\\wo GELU, \\w ReLU                   0.652161  \n",
       "\\wo spatial attention dropout       0.654153  "
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>dataset</th>\n",
       "      <th>audio_mous</th>\n",
       "      <th>brennan2019</th>\n",
       "      <th>broderick2019</th>\n",
       "      <th>gwilliams2022</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>reference</th>\n",
       "      <td>0.012990</td>\n",
       "      <td>0.018229</td>\n",
       "      <td>0.015285</td>\n",
       "      <td>0.026932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo clamping brain signal</th>\n",
       "      <td>0.000688</td>\n",
       "      <td>0.009408</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.019061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo final convs</th>\n",
       "      <td>0.013111</td>\n",
       "      <td>0.014835</td>\n",
       "      <td>0.011719</td>\n",
       "      <td>0.026580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo skip connections</th>\n",
       "      <td>0.013221</td>\n",
       "      <td>0.015965</td>\n",
       "      <td>0.011245</td>\n",
       "      <td>0.028014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo subject-specific layer</th>\n",
       "      <td>0.015870</td>\n",
       "      <td>0.013817</td>\n",
       "      <td>0.008554</td>\n",
       "      <td>0.027730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo initial 1x1 conv.</th>\n",
       "      <td>0.013233</td>\n",
       "      <td>0.015440</td>\n",
       "      <td>0.013500</td>\n",
       "      <td>0.027369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo non-residual GLU conv.</th>\n",
       "      <td>0.012960</td>\n",
       "      <td>0.003823</td>\n",
       "      <td>0.007538</td>\n",
       "      <td>0.026894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\w subj. embedding*</th>\n",
       "      <td>0.013135</td>\n",
       "      <td>0.019236</td>\n",
       "      <td>0.015858</td>\n",
       "      <td>0.026313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo spatial attention</th>\n",
       "      <td>0.013168</td>\n",
       "      <td>0.014461</td>\n",
       "      <td>0.013219</td>\n",
       "      <td>0.027978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo GELU, \\w ReLU</th>\n",
       "      <td>0.013420</td>\n",
       "      <td>0.016774</td>\n",
       "      <td>0.013479</td>\n",
       "      <td>0.027318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo spatial attention dropout</th>\n",
       "      <td>0.012962</td>\n",
       "      <td>0.017581</td>\n",
       "      <td>0.013773</td>\n",
       "      <td>0.027228</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "dataset                        audio_mous  brennan2019  broderick2019  \\\n",
       "row_label                                                               \n",
       "reference                        0.012990     0.018229       0.015285   \n",
       "\\wo clamping brain signal        0.000688     0.009408       0.000019   \n",
       "\\wo final convs                  0.013111     0.014835       0.011719   \n",
       "\\wo skip connections             0.013221     0.015965       0.011245   \n",
       "\\wo subject-specific layer       0.015870     0.013817       0.008554   \n",
       "\\wo initial 1x1 conv.            0.013233     0.015440       0.013500   \n",
       "\\wo non-residual GLU conv.       0.012960     0.003823       0.007538   \n",
       "\\w subj. embedding*              0.013135     0.019236       0.015858   \n",
       "\\wo spatial attention            0.013168     0.014461       0.013219   \n",
       "\\wo GELU, \\w ReLU                0.013420     0.016774       0.013479   \n",
       "\\wo spatial attention dropout    0.012962     0.017581       0.013773   \n",
       "\n",
       "dataset                        gwilliams2022  \n",
       "row_label                                     \n",
       "reference                           0.026932  \n",
       "\\wo clamping brain signal           0.019061  \n",
       "\\wo final convs                     0.026580  \n",
       "\\wo skip connections                0.028014  \n",
       "\\wo subject-specific layer          0.027730  \n",
       "\\wo initial 1x1 conv.               0.027369  \n",
       "\\wo non-residual GLU conv.          0.026894  \n",
       "\\w subj. embedding*                 0.026313  \n",
       "\\wo spatial attention               0.027978  \n",
       "\\wo GELU, \\w ReLU                   0.027318  \n",
       "\\wo spatial attention dropout       0.027228  "
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>dataset</th>\n",
       "      <th>audio_mous</th>\n",
       "      <th>brennan2019</th>\n",
       "      <th>broderick2019</th>\n",
       "      <th>gwilliams2022</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>reference</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo clamping brain signal</th>\n",
       "      <td>-0.660927</td>\n",
       "      <td>-0.116641</td>\n",
       "      <td>-0.124394</td>\n",
       "      <td>-0.443271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo final convs</th>\n",
       "      <td>-0.000643</td>\n",
       "      <td>-0.067200</td>\n",
       "      <td>-0.022593</td>\n",
       "      <td>-0.016426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo skip connections</th>\n",
       "      <td>-0.020980</td>\n",
       "      <td>-0.014944</td>\n",
       "      <td>-0.024662</td>\n",
       "      <td>-0.042648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo subject-specific layer</th>\n",
       "      <td>-0.250979</td>\n",
       "      <td>-0.055527</td>\n",
       "      <td>-0.065257</td>\n",
       "      <td>-0.222657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo initial 1x1 conv.</th>\n",
       "      <td>-0.046902</td>\n",
       "      <td>-0.036701</td>\n",
       "      <td>-0.015607</td>\n",
       "      <td>-0.028034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo non-residual GLU conv.</th>\n",
       "      <td>-0.005949</td>\n",
       "      <td>-0.197409</td>\n",
       "      <td>-0.065515</td>\n",
       "      <td>-0.005290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\w subj. embedding*</th>\n",
       "      <td>-0.027571</td>\n",
       "      <td>0.036562</td>\n",
       "      <td>0.012686</td>\n",
       "      <td>0.000136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo spatial attention</th>\n",
       "      <td>-0.016688</td>\n",
       "      <td>-0.051694</td>\n",
       "      <td>-0.015181</td>\n",
       "      <td>-0.048821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo GELU, \\w ReLU</th>\n",
       "      <td>-0.017238</td>\n",
       "      <td>-0.011799</td>\n",
       "      <td>-0.008898</td>\n",
       "      <td>-0.017708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo spatial attention dropout</th>\n",
       "      <td>-0.000645</td>\n",
       "      <td>0.010746</td>\n",
       "      <td>-0.010932</td>\n",
       "      <td>-0.015715</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "dataset                        audio_mous  brennan2019  broderick2019  \\\n",
       "row_label                                                               \n",
       "reference                        0.000000     0.000000       0.000000   \n",
       "\\wo clamping brain signal       -0.660927    -0.116641      -0.124394   \n",
       "\\wo final convs                 -0.000643    -0.067200      -0.022593   \n",
       "\\wo skip connections            -0.020980    -0.014944      -0.024662   \n",
       "\\wo subject-specific layer      -0.250979    -0.055527      -0.065257   \n",
       "\\wo initial 1x1 conv.           -0.046902    -0.036701      -0.015607   \n",
       "\\wo non-residual GLU conv.      -0.005949    -0.197409      -0.065515   \n",
       "\\w subj. embedding*             -0.027571     0.036562       0.012686   \n",
       "\\wo spatial attention           -0.016688    -0.051694      -0.015181   \n",
       "\\wo GELU, \\w ReLU               -0.017238    -0.011799      -0.008898   \n",
       "\\wo spatial attention dropout   -0.000645     0.010746      -0.010932   \n",
       "\n",
       "dataset                        gwilliams2022  \n",
       "row_label                                     \n",
       "reference                           0.000000  \n",
       "\\wo clamping brain signal          -0.443271  \n",
       "\\wo final convs                    -0.016426  \n",
       "\\wo skip connections               -0.042648  \n",
       "\\wo subject-specific layer         -0.222657  \n",
       "\\wo initial 1x1 conv.              -0.028034  \n",
       "\\wo non-residual GLU conv.         -0.005290  \n",
       "\\w subj. embedding*                 0.000136  \n",
       "\\wo spatial attention              -0.048821  \n",
       "\\wo GELU, \\w ReLU                  -0.017708  \n",
       "\\wo spatial attention dropout      -0.015715  "
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deltas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>dataset</th>\n",
       "      <th>audio_mous</th>\n",
       "      <th>brennan2019</th>\n",
       "      <th>broderick2019</th>\n",
       "      <th>gwilliams2022</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>reference</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo clamping brain signal</th>\n",
       "      <td>1.781294e-17</td>\n",
       "      <td>1.629815e-09</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>1.490116e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo final convs</th>\n",
       "      <td>5.500196e-01</td>\n",
       "      <td>1.774170e-07</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>1.490116e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo skip connections</th>\n",
       "      <td>7.027166e-13</td>\n",
       "      <td>5.513315e-02</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>1.490116e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo subject-specific layer</th>\n",
       "      <td>1.781185e-17</td>\n",
       "      <td>7.964151e-05</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>1.490116e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo initial 1x1 conv.</th>\n",
       "      <td>1.838185e-17</td>\n",
       "      <td>4.170742e-05</td>\n",
       "      <td>0.001694</td>\n",
       "      <td>1.490116e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo non-residual GLU conv.</th>\n",
       "      <td>6.922650e-03</td>\n",
       "      <td>4.656613e-10</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>4.098319e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\w subj. embedding*</th>\n",
       "      <td>6.841807e-13</td>\n",
       "      <td>1.564149e-05</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>8.779178e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo spatial attention</th>\n",
       "      <td>3.508152e-08</td>\n",
       "      <td>1.247972e-07</td>\n",
       "      <td>0.000164</td>\n",
       "      <td>1.490116e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo GELU, \\w ReLU</th>\n",
       "      <td>1.216311e-11</td>\n",
       "      <td>4.842899e-02</td>\n",
       "      <td>0.007145</td>\n",
       "      <td>1.490116e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo spatial attention dropout</th>\n",
       "      <td>8.507210e-01</td>\n",
       "      <td>9.695307e-02</td>\n",
       "      <td>0.002022</td>\n",
       "      <td>1.043081e-06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "dataset                          audio_mous   brennan2019  broderick2019  \\\n",
       "row_label                                                                  \n",
       "reference                      1.000000e+00  1.000000e+00       1.000000   \n",
       "\\wo clamping brain signal      1.781294e-17  1.629815e-09       0.000004   \n",
       "\\wo final convs                5.500196e-01  1.774170e-07       0.000019   \n",
       "\\wo skip connections           7.027166e-13  5.513315e-02       0.000019   \n",
       "\\wo subject-specific layer     1.781185e-17  7.964151e-05       0.000004   \n",
       "\\wo initial 1x1 conv.          1.838185e-17  4.170742e-05       0.001694   \n",
       "\\wo non-residual GLU conv.     6.922650e-03  4.656613e-10       0.000004   \n",
       "\\w subj. embedding*            6.841807e-13  1.564149e-05       0.000019   \n",
       "\\wo spatial attention          3.508152e-08  1.247972e-07       0.000164   \n",
       "\\wo GELU, \\w ReLU              1.216311e-11  4.842899e-02       0.007145   \n",
       "\\wo spatial attention dropout  8.507210e-01  9.695307e-02       0.002022   \n",
       "\n",
       "dataset                        gwilliams2022  \n",
       "row_label                                     \n",
       "reference                       1.000000e+00  \n",
       "\\wo clamping brain signal       1.490116e-08  \n",
       "\\wo final convs                 1.490116e-07  \n",
       "\\wo skip connections            1.490116e-08  \n",
       "\\wo subject-specific layer      1.490116e-08  \n",
       "\\wo initial 1x1 conv.           1.490116e-08  \n",
       "\\wo non-residual GLU conv.      4.098319e-02  \n",
       "\\w subj. embedding*             8.779178e-01  \n",
       "\\wo spatial attention           1.490116e-08  \n",
       "\\wo GELU, \\w ReLU               1.490116e-08  \n",
       "\\wo spatial attention dropout   1.043081e-06  "
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pvalues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>row_label</th>\n",
       "      <th>reference</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_label</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>reference</th>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo clamping brain signal</th>\n",
       "      <td>1.937180e-30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo final convs</th>\n",
       "      <td>5.296545e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo skip connections</th>\n",
       "      <td>9.782553e-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo subject-specific layer</th>\n",
       "      <td>8.523572e-30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo initial 1x1 conv.</th>\n",
       "      <td>7.723849e-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo non-residual GLU conv.</th>\n",
       "      <td>4.289443e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\w subj. embedding*</th>\n",
       "      <td>5.663648e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo spatial attention</th>\n",
       "      <td>7.156011e-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo GELU, \\w ReLU</th>\n",
       "      <td>3.731064e-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\wo spatial attention dropout</th>\n",
       "      <td>1.080691e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "row_label                         reference\n",
       "row_label                                  \n",
       "reference                      1.000000e+00\n",
       "\\wo clamping brain signal      1.937180e-30\n",
       "\\wo final convs                5.296545e-10\n",
       "\\wo skip connections           9.782553e-21\n",
       "\\wo subject-specific layer     8.523572e-30\n",
       "\\wo initial 1x1 conv.          7.723849e-28\n",
       "\\wo non-residual GLU conv.     4.289443e-14\n",
       "\\w subj. embedding*            5.663648e-03\n",
       "\\wo spatial attention          7.156011e-21\n",
       "\\wo GELU, \\w ReLU              3.731064e-17\n",
       "\\wo spatial attention dropout  1.080691e-01"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pvalues_agg_dset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}