AP-GCN_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import yaml\n",
    "import torch\n",
    "import logging\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.sparse as sp\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import torch.nn.functional as F\n",
    "import seaborn as sns\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "from torch.optim import Adam, Optimizer\n",
    "from collections import defaultdict\n",
    "from torch_geometric.data import Data, InMemoryDataset\n",
    "\n",
    "from model import APGCN\n",
    "from seeds import test_seeds, gen_seeds, quick_seeds\n",
    "from data import get_dataset, set_train_val_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_obj(obj, name):\n",
    "    with open('results/'+ name + '.pkl', 'wb') as f:\n",
    "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "def load_obj(name):\n",
    "    with open('results/' + name + '.pkl', 'rb') as f:\n",
    "        return pickle.load(f)\n",
    "\n",
    "def summary(results):\n",
    "    report={}\n",
    "    for k, v in results.items():\n",
    "        if k != 'steps' and k != 'probs':\n",
    "            boots_series = sns.algorithms.bootstrap(results[k], func=np.mean, n_boot=1000)\n",
    "            report[k] = np.mean(results[k])\n",
    "            report[f'{k}_ci'] = np.max(np.abs(sns.utils.ci(boots_series, 95) - report[k]))\n",
    "        else:\n",
    "            array = np.array([k.mean().cpu().detach().numpy() for k in results['steps']])\n",
    "            boots_series = sns.algorithms.bootstrap(array, func=np.mean, n_boot=1000)\n",
    "            report[k] = np.mean(array)\n",
    "            report[f'{k}_ci'] = np.max(np.abs(sns.utils.ci(boots_series, 95) - report[k]))\n",
    "    return report\n",
    "\n",
    "def plot_density(results):\n",
    "    fig, ax = plt.subplots()\n",
    "\n",
    "    z =[(x.cpu().numpy()).astype(int) for x in results['steps']]\n",
    "    z = np.vstack(z)\n",
    "    z = np.mean(z,axis=0)\n",
    "\n",
    "    sns.distplot(z, hist = False, kde = True,\n",
    "                 kde_kws = {'shade': True, 'linewidth': 3}, \n",
    "                 ax=ax)\n",
    "    plt.xlabel('Number of Steps')\n",
    "    plt.ylabel('Density')\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model: torch.nn.Module, optimizer: Optimizer, data: Data, train_halt, weight_decay: float):\n",
    "    model.train()\n",
    "    \n",
    "    for param in model.prop.parameters():\n",
    "        param.requires_grad = train_halt\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    logits, steps, reminders = model(data)\n",
    "    \n",
    "    loss = F.nll_loss(logits[data.train_mask], data.y[data.train_mask])\n",
    "    l2_reg = sum((torch.sum(param ** 2) for param in model.reg_params))\n",
    "    loss += weight_decay/2 * l2_reg + model.prop_penalty *(\n",
    "            steps[data.train_mask] + reminders[data.train_mask]).mean()\n",
    "    \n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    return\n",
    "\n",
    "def evaluate(model: torch.nn.Module, data: Data, test: bool, weight_decay: float):\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        logits, steps, reminders = model(data)\n",
    "        \n",
    "        loss = F.nll_loss(logits[data.train_mask], data.y[data.train_mask])\n",
    "        l2_reg = sum((torch.sum(param ** 2) for param in model.reg_params))\n",
    "        loss += weight_decay/2 * l2_reg + model.prop_penalty *(\n",
    "                steps[data.train_mask] + reminders[data.train_mask]).mean()\n",
    "\n",
    "    eval_dict = {}\n",
    "    keys = ['train','val', 'test'] if test else ['val']\n",
    "    eval_dict['steps'] = steps\n",
    "    for key in keys:\n",
    "        mask = data[f'{key}_mask']\n",
    "        pred = logits[mask].max(1)[1]\n",
    "        acc = pred.eq(data.y[mask]).sum().item() / mask.sum().item()\n",
    "        eval_dict[f'{key}_acc'] = acc\n",
    "    return eval_dict, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run(dataset: InMemoryDataset,\n",
    "        model: torch.nn.Module,\n",
    "        seeds: np.ndarray,\n",
    "        test: bool = False,\n",
    "        max_epochs: int = 10000,\n",
    "        patience: int = 100,\n",
    "        lr: float = 0.01,\n",
    "        weight_decay: float = 0.01,\n",
    "        num_development: int = 1500,\n",
    "        device: str = 'cuda'):\n",
    "    \n",
    "\n",
    "    best_dict = defaultdict(list)\n",
    "\n",
    "    for seed in tqdm(seeds):\n",
    "        for _ in range(config['niter_per_seed']):\n",
    "            torch_seed = gen_seeds()\n",
    "            torch.manual_seed(seed=torch_seed)\n",
    "            \n",
    "            dataset.data = set_train_val_test_split(\n",
    "                seed,\n",
    "                dataset.data,\n",
    "                num_development=num_development,\n",
    "                num_per_class=20\n",
    "                ).to(device)\n",
    "            \n",
    "            model.to(device).reset_parameters()\n",
    "            optimizer = Adam(model.parameters(),lr=lr)\n",
    "            \n",
    "            patience_counter = 0\n",
    "            best_loss = 999\n",
    "            tmp_dict = {'val_acc': 0}\n",
    "            \n",
    "            start_time = time.perf_counter()\n",
    "            for epoch in range(1, max_epochs + 1):\n",
    "                if patience_counter == patience:\n",
    "                    break\n",
    "\n",
    "                train(model, optimizer, dataset.data, epoch%5==0, weight_decay)\n",
    "                eval_dict, loss = evaluate(model, dataset.data, test, weight_decay)\n",
    "                                \n",
    "                if(eval_dict['val_acc'] > tmp_dict['val_acc']) or (\n",
    "                  (eval_dict['val_acc'] == tmp_dict['val_acc']) and loss < best_loss):\n",
    "                    patience_counter = 0\n",
    "                    tmp_dict['epoch'] = epoch\n",
    "                    tmp_dict['runtime'] = time.perf_counter() - start_time\n",
    "                    \n",
    "                    for k, v in eval_dict.items():\n",
    "                        tmp_dict[k] = v\n",
    "                else:\n",
    "                    patience_counter += 1\n",
    "                    \n",
    "                if loss < best_loss:\n",
    "                    best_loss = loss\n",
    "                    patience_counter = 0\n",
    "                            \n",
    "            print(\"Epoch: {:.1f}\"\" Train: {:.2f}\"\" Val: {:.2f}\"\" Test: {:.2f}\".format(\n",
    "                  tmp_dict['epoch'],\n",
    "                  tmp_dict['train_acc'] * 100,\n",
    "                  tmp_dict['val_acc'] * 100,\n",
    "                  tmp_dict['test_acc'] * 100))\n",
    "                \n",
    "            for k, v in tmp_dict.items():\n",
    "                best_dict[k].append(v)\n",
    "            \n",
    "\n",
    "    return dict(best_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/PhD/code/AP-GCN/sparsegraph.py:393: UserWarning: 124 self loops removed\n",
      "  warnings.warn(\"{0} self loops removed\".format(num_self_loops))\n"
     ]
    }
   ],
   "source": [
    "device = 'cuda'\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.synchronize()\n",
    "\n",
    "config = {'dataset_name': 'citeseer',#'cora_ml' 'pubmed' 'ms_academic'\n",
    "          'test': True,\n",
    "          'use_lcc': True,\n",
    "          'num_development': 1500,\n",
    "          'niter_per_seed': 5,\n",
    "          'hidden_units': 64,\n",
    "          'lr': 0.01,\n",
    "          'dropout': 0.5,\n",
    "          'weight_decay': 8e-03\n",
    "         }\n",
    "    \n",
    "dataset = get_dataset(\n",
    "    name=config['dataset_name'],\n",
    "    use_lcc=config['use_lcc']\n",
    "    )\n",
    "\n",
    "dataset.data = dataset.data.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b0f52827d934236bbb192d23e63df4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 978.0 Train: 93.33 Val: 78.40 Test: 78.52\n",
      "Epoch: 368.0 Train: 92.50 Val: 78.80 Test: 78.03\n",
      "Epoch: 288.0 Train: 90.83 Val: 78.80 Test: 78.36\n",
      "Epoch: 194.0 Train: 88.33 Val: 79.00 Test: 76.72\n",
      "Epoch: 301.0 Train: 90.83 Val: 78.80 Test: 77.54\n",
      "Epoch: 90.0 Train: 78.33 Val: 77.20 Test: 74.75\n",
      "Epoch: 82.0 Train: 80.83 Val: 76.40 Test: 74.43\n",
      "Epoch: 125.0 Train: 83.33 Val: 77.80 Test: 76.72\n",
      "Epoch: 186.0 Train: 87.50 Val: 76.40 Test: 76.23\n",
      "Epoch: 131.0 Train: 82.50 Val: 77.40 Test: 77.21\n",
      "\n",
      "FINAL\n",
      "Train Accuracy: 86.83 ± 3.17%\n",
      "Stopping Accuracy: 77.90 ± 0.58%\n",
      "Test     Accuracy: 76.85 ± 0.87%\n",
      "Steps: 6.81 ± 0.88\n",
      "Epochs:  274.30 ± 178.46\n",
      "Runtime: 12.4148 ± 7.9651\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3xc9Z3v/9dHo14tq1m2jDvYBgwY00kDkkBIIJuQQtoWbhL2hiyb7N4bUm5+m/zu3rTN3s1u2N2QQAgloSRAIDghC6EabFwwNsZdLnKXi3rXfO8fZzQ6ErI0kmd0pryfj4ceOudoygdjz1vfcr5fc84hIiKSbLKCLkBERGQkCigREUlKCigREUlKCigREUlKCigREUlK2UEXMF6VlZVu9uzZQZchIiJxsnbt2qPOuarh11MuoGbPns2aNWuCLkNEROLEzPaMdF1dfCIikpQUUCIikpQUUCIikpQUUCIikpQUUCIikpQUUCIikpQUUCIikpQUUCIikpRS7kZdEZHJ8PjrB/i3Z7aTm51FbVkB06fks6RuCh86bwZZWRZ0eRlBASUiMkx9Yxt///Dr9PSFAdh0oCXykz20dvXyl5fNCa64DKIuPhERH+ccX3/0jWg4DXf7szvp6u2f5KoykwJKRMTn12v38Ur9MQCyDG69cgG3XrmAqUW5ABxt6+bB1Q1BlpgxFFAiIhHH2rr5x+Wbo+fvO7uWi+dWcPHcCj6wpDZ6/T+f30l3n1pRiaaAEhGJ+McnN9PU0QtAZXEuH15aF/3ZFQtrKCvIAeBgcxe/Wbs/kBoziQJKRARY39DEI68Nhs5Nl88hPycUPc/NzuL9vlbUvz+3g97+kcepJD4UUCIiwCPr9kWPL5ozlXNnlr/lMVctqqE4z5v8vO9EJ4+9plZUIimgRCTj9fWHWb7xYPT8PYtrRnxcfk6I95092Iq644X6hNeWyRRQIpLxXqk/xtG2HgDKC3NYOK30pI9975k15Ia8j87tR9o40NQ5KTVmIgWUiGS8x9cfiB5fPLdi1JUiCnOzOX1aSfR81a5jCa0tkymgRCSjdff184dNh6Lnl86rGPM5i2sHW1grdx5PSF2igBKRDPf81kZau/oAqC7JY15V8ZjPWVQ72IJaqRZUwiigRCSjPbFhcHLEJfMqMBt7Idh5VcXRcag9xzo42KxxqERQQIlIxuro6ePpNw9Hzy+dVxnT83JCWZxeM9jSWlWvbr5EUECJSMZ6evMROiMLv86YUsDM8oKYn7vIPw5Vr26+REhoQJnZ1Wa21cx2mNltozzuw2bmzGxZIusREfH73euDs/cujbF7b4B/osSqXWpBJULCAsrMQsDtwDXAYuBGM1s8wuNKgFuBVYmqRURkuN7+MC/vHGz5XDR37Nl7fvOqi8kJeYG262g7h5q74lqfJLYFdSGwwzlX75zrAR4Arh/hcf8/8D1A/3dFZNKsb2iirdubvVdZnMv0svxxPT8nlMWCat0PlUiJDKgZgH/TlH2Ra1FmthSY6Zx7crQXMrPPmdkaM1vT2NgY/0pFJOO8tP1o9PjsGWXj6t4bsHi6xqESKbBJEmaWBfwz8HdjPdY5d4dzbplzbllVVVXiixORtPfSjqEBNRH+iRKayRd/iQyo/cBM33ld5NqAEuAs4Dkz2w1cDDyuiRIikmgtXb2sb2iKnp85wYCaXzU4DlV/tJ3DLRqpiKdEBtRqYIGZzTGzXODjwOMDP3TONTvnKp1zs51zs4GVwHXOuTUJrElEhJU7j9EfdgDMriikND9nQq+Tmz10HOpVzeaLq4QFlHOuD7gFeArYDDzknNtkZt82s+sS9b4iImNZEYfuvQHzqwdv2N1yqOWUXkuGyk7kizvnlgPLh1375kke+85E1iIiMuBFf0DVTTml1zptamH0eOuhtlN6LRlKK0mISEY50NRJfWM7ADkh44yakjGeMbo63+oTWw+rBRVPCigRySj+6eULp5WSm31qH4PTpxQwsH1Uw/FO2iP3VsmpU0CJSEbxd++ddYrjT+DdsFtbNtiK2n5E3XzxooASkYwRDjtejuMEiQH+br5th1rj8pqigBKRDLLlUCvH2nsAKM3PZlZF4RjPiM2QiRKHFVDxooASkYzhXy9vUW0pWRNY3mgkM8v9M/kUUPGigBKRjOFfjsi/Xcapqpvqn8mngIoXBZSIZATnHK/uHgyoRXEMqJqS/OiSR42t3RyPdCPKqVFAiUhG2HGkLRocxXnZzBjH7rljycoy6nzdfNvUiooLBZSIZISVvnXyFk4ridv40wD/dvEKqPhQQIlIRlhVP3SCRLzN9M3k26KJEnGhgBKRtOecG7LSeCICakgXnwIqLhRQIpL2dh/r4EhrNwCFuSFmTY3P/U9+M8uHzuRzzsX9PTKNAkpE0p6/e++MmhKysuI7/gQwtSiXwtwQAK1dfRzS5oWnTAElImkv0d17AGamG3bjTAElImlv1SQEFMBM/w27CqhTpoASkbTWcLyD/U2dAORlZzG7Mv7jTwOGtKA01fyUKaBEJK35W09n1JSQnZW4j7063+SL7Ye17capUkCJSFp7dVdi73/ym16WHz3efaxdM/lOkQJKRNKavwW1sPbUtncfS1lBDvk53sdqa1cfJzp6E/p+6U4BJSJp61BzF3uOdQCQEzLmVRUn9P3MjJqSoa0omTgFlIikLf/+T6fXlJATSvxHXo2vm2+PAuqUKKBEJG2tTPD6eyOZVuprQR3tmJT3TFcKKBFJW/4NCicroGpK1YKKFwWUiKSlIy1d1B/1AiInZMxP8PjTgGmledHj3cfUgjoVCigRSUv+2XvzqorJzZ6cjzu1oOJHASUiack/QWLxJHXvAZQX5Ua3fz/R0UuzpppPmAJKRNJSEONPAFlmQ1tRx9WKmigFlIiknWNt3Ww/4i01FMoyFtRMzvjTgCEz+TQONWEKKBFJO68OGX8qIi87NKnvP6QFdVQtqIlSQIlI2pms7TVOpkYtqLhQQIlI2hlyg+60yQ+oaVpNIi4UUCKSVk6090T3Ysoyb4mjyVZTonuh4kEBJSJp5ZX6YwzscjGvqpiC3MkdfwKoKM4jlOVNNT/a1k1bd9+k15AOFFAiklZe2nE0enzWjLJAaghlGdW+VtRetaImRAElImnl5SQIKNCKEvGggBKRtNFwvCM65pMbymJB9eTe/+Sne6FOnQJKRNLGyzsHW08Laydn/6eTUQvq1CmgRCRtrNgxOL387AC79wCmlfln8imgJkIBJSJpIRx2rPCNP505PdiAGtqCUhffRCigRCQtbD3cyrH2HgCK87KZVVEYaD1VxXlEZppzsLmLrt7+QOtJRQooEUkLQ1tPpWSZBVgNZIeyqCwe7OZTK2r8FFAikhb8ARX0+NMAfzdfw3EF1HglNKDM7Goz22pmO8zsthF+frOZbTSz9Wb2kpktTmQ9IpKeevrCQxaIDfL+J78q3826DScUUOOVsIAysxBwO3ANsBi4cYQA+qVz7mzn3LnA94F/TlQ9IpK+1jc00dHjjfFUFecNabkEyR9Q+050BlhJakpkC+pCYIdzrt451wM8AFzvf4BzrsV3WgS4BNYjImnqxe2N0eNkaT2BF5YD1MU3ftkJfO0ZQIPvfB9w0fAHmdkXgC8DucAVI72QmX0O+BzAaaedFvdCRSS1/WnLkejxOXXJE1DVakGdksAnSTjnbnfOzQO+AnzjJI+5wzm3zDm3rKqqanILFJGkdqi5i00HvM6YkBlnJ1FAaQzq1CQyoPYDM33ndZFrJ/MA8MEE1iMiacjfelpUW0JhbiI7hsanrCCHnJA33b21q4/mzt6AK0otiQyo1cACM5tjZrnAx4HH/Q8wswW+02uB7QmsR0TS0J+2HI4en3daeYCVvJWZDW1FaRxqXBIWUM65PuAW4ClgM/CQc26TmX3bzK6LPOwWM9tkZuvxxqH+PFH1iEj66ertH7L/09IkCyiA6pLBGYUahxqfhLaFnXPLgeXDrn3Td3xrIt9fRNLbKzuP0dUbBmB6WT7TypJjermffzWJfRqHGpfAJ0mIiEzUM0ncvTdAM/kmTgElIinJOcefNg9OkFh62pQAqzk5jUFNnAJKRFLSlkOtHGjuAqAwN8Tp00oCrmhkWk1i4hRQIpKSht6cO4XsrOT8OKsedi+Uc1owJ1bJ+X9URGQMT2/2jz8lZ/ceeHtT5ed4H7UdPf2c6NC9ULFKnjvaRCSunHPsbGxn3d4T9Icd+TlZFOSEqCzOY0ndFHKzU/f30/1Nnby2twmALINzZiZvQHn3QuVHx58ajncwtSg34KpSgwJKJM28tP0oj762nxU7jnKopWvExxTmhrhkbgVvP72K951dO2ScJBU8ueFA9Pis6WWU5ucEWM3YqorzogG170RnUgdqMlFAiaSJ5o5evvXEJh55bbQVxTwdPf08s+UIz2w5wnd+v5nPXDKbz799LhXFqRFUj78+GFCXzKsIsJLYDB+HktgooETSwNNvHuZrj27kSGv3kOuFuSEW15ZSkp9NT1+Y7r4we493DHlcV2+YO16o576Ve7jp8jnccsV88rJDk/2fELP6xjbe2O8tDpudZVwwe2rAFY1t6Ew+BVSsFFAiKe4/n9/Jd3+/Zci1S+dVcPWZ05hbVUwoy97ynEPNXWzY38SzW46w+5j3gdnR08+//WkHL2xr5D8+dT7TpxRMSv3j9cTrB6PH586cQlFe8n+MDb0XSlPNY5X8/2dF5KQefW3fkHAqK8jhpsvnjNmqmFaWz7Syabx7UQ1rdp/g4bUNNETu0Xl9XzPv/7eX+PGN53Hp/MqE1j9ezjkef32wC/PSFOjeA7WgJip1p/GIZLgVO47yP3+9IXq+uLaUf7rhnHF1eZkZF8yZync/vITPXDKLkHmtrePtPXzqzlXc+8ruOFd9ajYfbGVnYzsAedlZSbu80XBVxUNv1tW9ULFRQImkoM0HW7j53rX09nsfdDPLC/i795xOcf7EOkWyzLjmrFq+ce0iygq8GXFhB//rt5t4eE3DGM+ePP7JEefPKic/J3nHyvyK8rIpyvVq7e4L09jWPcYzBBRQIimntauXm+5eTWt3HwBTi3L5ytUL47JR38LaUv7Pn53NvKqi6LWv/GYDT206dMqvfaqcczzhC6hL5yVX9+NYNA41fgookRTzg6e2RtegK8gJ8ZWrF8Z1evjUoly+es0iZk0tBLyW1Bd/+RorfPsuBWHtnhPsb/I+2ItyQyxJoq3dY6FxqPGLKaDM7BEzu9bMFGgiAVq75wT3rtwTPb/p8jmcFgmSeCrKy+a2axYyrdTbX6mnP8zn7lnD1kOtcX+vWN2/am/0+MI5FeSEUuvjqEobF45brP+H/x34BLDdzL5rZmcksCYRGUFPX5ivPbKRgfH1c2dOSegstimFuXztfQujy/K09/Tz1/etpS3StTiZGlu7eXLD4PTyqxZVT3oNp8o/UULbbsQmpoByzj3tnPsksBTYDTxtZi+b2V+aWXKvMSKSJn76Yj1bD3stmLzsLP7qsjmYvfUep3iqKsnntqsXkhdZt6/+aDtf+c2GSZ+F9uDqvfT0ezvnLqguZm5V8aS+fzz4V5MY6KqU0cXcRjazCuAvgP8GvAb8CC+w/ishlYlI1O6j7fzome3R848umzlp6+fNnFrIZ982N3r+5IaD/OLl3ZPy3gC9/WHuWznYvfeeM6dN2nvHk/aFGr9Yx6AeBV4ECoEPOOeuc8496Jz7IpB6v8qIpJjvP7WFnj6vBTGnsoj3TvKH9GXzK7lqUU30/B+Xb2bd3hOT8t7/9ebh6KK3ZQU5XDwn+Zc2Gkmlr4tv/4lOwmHdCzWWWFtQP3XOLXbOfcc5dxDAzPIAnHPLEladiLBxXzPLNw5O877p8jkjLl+UaJ+5ZBZzK73p5739ji/+8jVauhK/t5G/tXblwmqyU2xyxICC3BDFkWWZevrDb1k3Ud4q1v/T/3uEa6/EsxARGdk//XFr9PiiOVOZF9D4S04oi7+9akH0htP9TZ1849E3EjoeteVQC6t2HQe8fZ+u9LXiUpGmmo/PqAFlZtPM7HygwMzOM7Olka934nX3iUgCrao/xvPbGgEwg4+cPzPQeqpK8oeMRz3++gEejWF7j4m6e8Xu6PEFs6em/EZ/1RqHGpexbj1/L97EiDrgn33XW4GvJagmEcFbOcHfenrb/EpmlAe/wvhFcyt4Z0MTz0WC83899gbnzypnVkXRGM8cnx1H2nh47b7oeapOjvBTC2p8Rm1BOed+4Zx7F/AXzrl3+b6uc849Mkk1imSk57Y1snq3NxEhlGV8eGldwBUN+vNLZ0dv4m3v6edvHlgfncQRL9/9/Rb6IxMJFteWsmhaSVxfPwjDF42V0Y3VxfepyOFsM/vy8K9JqE8kIznn+KGv9XTlwmqqS/NHecbkys8JccsV86Orn7/e0MR3fr85bq+/sv4YT28+HD3/1MWzEn7P12SoVBffuIw1SWKgzV4MlIzwJSIJ8NzWxuiusbmhLD543oyAK3qreVXFfOyCwTGxn6/Yze82HBjlGbEJhx3/Z/lg2L1tfiVzKuPbfRiUIatJqItvTKOOQTnnfhL5/q3JKUdEnHP8258Gb8q9YlE15YXJOTng/Utq2Xa4lTV7vK7Ir/x6AwunlTK/euIzDZ/YcIAN+5oByAkZH70g2Ikh8eQfgzrQ1El/2AVyy0CqiPVG3e+bWamZ5ZjZM2bW6Ov+E5E4eqX+GOv2NgHe2NP7z64NuKKTMzNufsc8akq9D96B9fraJ7heX2tXL9//w2DX5vvOrh1yg2uqy88JURrZs6u333GktSvgipJbrPdBvcc51wK8H28tvvnA/0hUUSKZ7PZnd0SP33F6VVy30kiEorxsvnTV6eSEvJbA9iNt3HzfWrp6+8f1Ov1hx9/86rXoOnWl+dlcd870uNcbNC15FLtYA2qgK/Ba4GHnXHOC6hHJaOv2nmDFjmOAd2NqqnxAz6oo4qbL50TPX9x+lM/fO76Q+s7yzTy7tTF6/plLZsdlE8ZkU1msqeaxijWgfmdmW4DzgWfMrApQ21Qkzm7/02Dr6dJ5ldQk0cy9sbzj9OohU+Gf39bIX9+3lu6+sUPqwdV7+dlLu6Ln1587ncvmp9aOubEa0oLSzrqjinW7jduAS4FlzrleoB24PpGFiWSaNw+08MyWI9Hz689NjdaT3w3n1/Eh34zDZ7c28uk7X2Xb4ZE3OuwPO+5ftYdvPPZG9NqyWeV8dFn6TIwYbsjW72pBjWo87eeFePdD+Z9zT5zrEclYtz832Hq6cPZU6spTczWxG86vI+zgsfXeEkiv7jrONT96kU9ceBpfvGI+UwpzyQkZr+46zreeeJM3D7ZEnztraiFfeNd8stLgnqeT0c26sYspoMzsXmAesB4YaK87FFAicVHf2MbyjYM7xibjfU+xMjM+uqyO7JDxm3X7cM5rKd27cs+Q7eqHqy7J4+/fewb5OaFJrHbyaZJE7GJtQS0DFrvJ3kZTJEP8x3M7o1u5n1NXlvI3ppp5SzMtm1XOvSv3sOlAy0kfmxvK4rpzp/P+JbXkZad3OIHuhRqPWAPqDWAacHCsB4rI+Oxv6hyyIvgHz03d1tNwsyqK+Pr7FrF2zwkeeW0/B5o66et39DtHdpZx0Zyp3HjhaUk/lT6e8rJDlBbk0NLZS1/Ycbili+lTgl8EOBnFGlCVwJtm9ioQ3WXLOXddQqoSySA/faGevsiiqAunlbCwtjTgiuLLzFg2eyrLZg/uhBsOOxxkbMuhqjiXlk5vs8d9JzoVUCcRa0D9QyKLEMlUja3d/OrVvdHz69Oo9TSarAwNpgFVJXnsbGwHoOF4Bxem6Db2iRZTQDnnnjezWcAC59zTZlYIpH9nsUiC3bViF92RbSrmVBZxTl1ZwBXJZNBMvtjEuhbfZ4FfAz+JXJoBPJaookQywYn2Hu55eXf0/Ppzp6fFlhIyNm1cGJtYV5L4AnAZ0ALgnNsOVCeqKJFMcNeKXbT3eHdtzJhSwAWz1c2TKapKBlcIUQvq5GINqG7nXM/ASeRm3TGnnJvZ1Wa21cx2mNltI/z8y2b2ppltiKySPiv20kVSV3NHL3ev2B09/9DSGWl9c6oMNaQF1aQW1MnEGlDPm9nXgAIzezfwMPDEaE8wsxBwO3ANsBi40cwWD3vYa3jLJy3B60L8/niKF0lVd63YRWtkS4rpU/K5eE5FwBXJZPKPQR1s6qKvPxxgNckr1oC6DWgENgKfB5YD3xjjORcCO5xz9ZHW1wMMW7/POfesc27g14eVQB0iaa65s5e7VgwujPpn59Vl/Ky2TJObnUVZQQ4AfWHHwWatvT2SWGfxhc3sMeAx51zjmE/wzAAafOf7gItGefxNwO9jfG2RlHX3it20dnmtp9qyfC6dq9ZTJqouyaM5ci9Uw/EOZk5NzbUXE2nUFpR5/sHMjgJbga2R3XS/Gc8iIrvzLgN+cJKff87M1pjZmsbGWPNRJPm0dvVy50v10fMPnjtDracMVe3bSmXvcY1DjWSsLr4v4c3eu8A5N9U5NxWvFXSZmX1pjOfuB/xr5tdFrg1hZlcBXweuc851D/85gHPuDufcMufcsqqqqjHeViR5/XzFbloiradppflpu+eRjK2mdHAcao8CakRjBdSngRudc9EOc+dcPfAp4DNjPHc1sMDM5phZLvBx4HH/A8zsPLx7q65zzh0Z4TVE0kZzZy8/fdHXejpvRsYu9SNQXaIW1FjGCqgc59zR4Rcj41A5oz3ROdcH3AI8BWwGHnLObTKzb5vZwBp+PwCKgYfNbL2ZPX6SlxNJeXe+WD9k7OlytZ4ymr8F1aCAGtFYkyR6JvgzAJxzy/Fm/PmvfdN3fNVYryGSDk6093CX776nDy+tU+spw/lbUHuOKaBGMlZAnWNmI23kYkD+CNdFZAQ/fbGetsh9TzOmFHCJZu5lvPLCHHJCRm+/o7mzl+aOXsoKR+2YyjijdvE550LOudIRvkqcc/qTFInBsbZu7vatuffhpbrvSbxtSDQONbpYb9QVkQm644V6OiJr7s0sL+CiuVpzTzw1mmo+KgWUSAIdbevmnlf2RM8/fH6d1tyTqOohU83bA6wkOSmgRBLojhfq6ez1Wk+zphZqxXIZoqZEM/lGo4ASSRCv9bQ7eq7Wkwyn1SRGp4ASSZA7Xqinq9dbpXpWRSHLZpUHXJEkmxpNNR+VAkokAd7Selpap91y5S38+0IdaOqkV9tuDKGAEkkAtZ4kFrnZWUwtygUg7GC/dtcdQgElEmfH2rq51zdz7wa1nmQU1b5WlMahhlJAicTZ3S/vHpy5V1HI+Wo9ySj890JpVfOhFFAicdTa1csvfKtGfPDcGWo9yaiqNdX8pBRQInH0q1f3Dtnv6ULd9yRjGLKahGbyDaGAEomT7r5+fvZidOs0PnDOdK25J2Pyt6DUxTeUAkokTn6zdj9HWr1NocsLc3jbAu33JGPzt6AajnfgnAuwmuSigBKJg77+MD95YWf0/H1n15IT0j8vGVtJfjb5Od7flbbuPo63j7nVXsbQvyCROPjDpkPRlQCK8kJcubAm4IokVZjZkBUlNNV8kAJKJA7ufGlw7Om9i6dRkBsKsBpJNf5VzRVQgxRQIqdofUMTr+1tAiA7y3j3YrWeZHw0k29kCiiRU/TzFYOtp0vmVTClMDfAaiQV+QNq11HtCzVAASVyCg63dPHkhoPR82vOqg2wGklV06cURI93NrYFWElyUUCJnIL7V+6hL+xNCz6jpoQ5lUUBVySpaHrZYAtqZ2O7pppHKKBEJqirt5/7V+2Nnl9z1rQAq5FUVlaQQ2FkYk1bd1/0frpMp4ASmaAnXj/Ascg9K5XFuSzTskYyQWambr4RKKBEJsA5x92+RWHfvXgaIS1rJKdgeDefKKBEJmR9QxObDrQAkBvK4oozqgOuSFJdrb8FdUQtKFBAiUzIL31jT5fMq6A4PzvAaiQdzCgbDKh6TTUHFFAi49bc2csTGw5Ez69apNaTnLrpakG9hQJKZJwee20/Xb1hAGZNLWReVXHAFUk6qCnNY2AYc39TJ509/cEWlAQUUCLj4Jzj/lV7oudXLqrWjrkSF9mhLKp9i8bWH1UrSgElMg5r95xg22HvgyMvO4vL5mvPJ4kffzdfvWbyKaBExsM/OeLSeZUU5mpyhMTP9Cn+qeZqQSmgRGJ0or2H320cXHfvSk2OkDibXua/WVctKAWUSIwefW0/PX3e5Ig5lUWaHCFxN7SLTy0oBZRIDJxzPLi6IXp+xUK1niT+/F189Y3thMOZvWisAkokBusbmth6uBXwJkdcOq8i4IokHZXk51Cc541rdvb2c7ClK+CKgqWAEomBv/V08dwKTY6QhJmhG3ajFFAiY2jr7uPx1wdXjlD3niTS0G4+BZSIjOJ3rx+gI3JX/4wpBSyo1uQISZxazeSLUkCJjOGBYZMjtHKEJJL2hRqkgBIZxZZDLaxvaAIglGVcvkArR0hi6WbdQQookVH4J0dcMLuc0vycAKuRTFBdkk9OyGulH27p5kRk1+ZMpIASOYmu3n4eWbc/ev4ubUookyCUZcwsL4yeD2yMmYkSGlBmdrWZbTWzHWZ22wg/f7uZrTOzPjO7IZG1iIzX8o0Hae7sBaC6JI+zZpQFXJFkitmVRdHjTQeaA6wkWAkLKDMLAbcD1wCLgRvNbPGwh+0F/gL4ZaLqEJko/8Kw71pYTZYmR8gkmV2hFhRAIu82vBDY4ZyrBzCzB4DrgTcHHuCc2x35WTiBdYiM27bDrazZcwKAkBnvPL0q4Iokk8yuUAsKEtvFNwNo8J3vi1wbNzP7nJmtMbM1jY2NcSlOZDT+1tOy2eVMKcwNsBrJNKdVFDLQYK8/2k5HT1+wBQUkJSZJOOfucM4tc84tq6rSb7KSWN7kiH3Rc60cIZMtLzsU3XrDOdh8sDXgioKRyIDaD8z0nddFrokktd9tOEhLl/cbqyZHSFD841BvZmg3XyIDajWwwMzmmFku8HHg8QS+n0hc/HLVnujxlZocIQEZOpMvMydKJCygnHN9wC3AU8Bm4CHn3CYz+7aZXQdgZheY2T7gI8BPzGxTouoRicXmgy2s26ns4OwAABIpSURBVDu4csQ7dO+TBGToRInMDKiE7hngnFsOLB927Zu+49V4XX8iSeHnK3ZFjy+YXU5ZgVaOkGD4A2rroVZ6+8PkhFJi2kDcZNZ/rcgojrV189j6wW01rjmrNsBqJNMV52dTWezNHu3pD7MjA/eGUkCJRPxy1V56+rxb8uZVFWlbDQmcvxX1ZgZ28ymgRICevjD3rBycHHHNWbXaVkMCNyvDx6EUUCLAkxsP0NjaDUB5YQ4XzZkacEUiw5c8yryp5gooyXjOOe56aXf0/D2Lp5GdYYPRkpz8U83fPNiCcy7Aaiaf/hVKxlu75wQb93u/neaEjCsWaWq5JIeKolyK87zJ1q1dfTQc7wy4osmlgJKM95/P10eP37agSpsSStIwsyHdfG9kWDefAkoy2hv7m3l68+Ho+dVnTguwGpG3mls1OJv01V3HA6xk8imgJKP9y9Pbo8cXzZnKzKmFozxaZPItri2NHq+sPxZgJZNPASUZa+O+wdaTAR9eqkVNJPmcMa2EUOSWhy2HWjnW1h1wRZNHASUZ60fPbIseXzRXrSdJTvk5IeZVD87my6RuPgWUZKQN+5p4evMRwGs9feg8tZ4kefm7+V7JoG4+BZRkJP/Y08VzK9R6kqS2ePrgnmSv7FRAiaStFTuO8qctvtbT0hnBFiQyhtNriglleeNQ24+0cTRDxqEUUJJRevrCfPO3b0TPL5tfSV25Wk+S3PKyQ8z3TTfPlNl8CijJKD9fsYudje0AFOSE+MRFpwVckUhszpzuG4fKkG4+BZRkjIPNnfzomcGxpxvOr6O8MDfAikRitygD74dSQEnG+N9Pbqajpx+AmeUFvOfMmoArEond6TUlZEfGoXY2tnOkpSvgihJPASUZ4fltjTy54WD0/C8vm0N2lv76S+rIzc5iQc3gOFQmTDfXv1BJe0fbuvn7h1+Pnl82v3JId4lIqhi67FH637CrgJK0Fg47/u6h16ObEZYW5PApTYyQFOW/H+q5rUcIh9N7fygFlKS1n71Uz/PbGqPnX3jnPKZoYoSkqNNriqP7Qx1s7uLV3endilJASdpa39DE9/+wNXr+gSW1LKmbEmBFIqcmOyuLi+dWRM9/u35/gNUkngJK0tKh5i7++31r6Yt0gcyrKuKjF8wMuCqRU3f5/Mro8fKNh+ju6w+wmsRSQEnaae7s5c/vepUDzd403IKcEF+8YoFm7UlaOL2mmKriPMD7u/781sYxnpG69C9W0kpXbz+fvWcNWw+3AhAy49YrF1BTmh9wZSLxYWZcNt/fzXcgwGoSSwElaaOvP8yXHlw/ZL+cz79jLufM1LiTpJfLfN18T28+TGtXb4DVJI4CStJCV28/N9+3jt+/cSh67RMXnsbbFlQFWJVIYtSVFzKrwlvkuLsvzB98f+/TiQJKUl5zRy+fvnNVdPt2gGvOmsb7l9QGWJVIYvknS6RrN58CSlLagaZOPvKTl1m9+0T02geW1PLpi2dhZgFWJpJYl8ytYOBv+Ms7j3KgqTPQehJBASUp65nNh7n2X19k2+G26LVPXzyLT1ykcJL0V1GcF12yK+zgx8/uCLii+FNAScrp7uvnW09s4qZfrOFEhzc4HDLjC++az/vOVreeZI7rz50ePX5wdQO7jrYHWE38KaAkpayqP8b1P17Bz1fsjl6bWpTL169dNKRPXiQTnD2jLLqAbH/Y8cM/bh3jGalFASUpYd+JDr5w/zo+dsdKthxqjV5felo53/3Q2VqdXDKSmXHjhYMrpPxuw0He2N8cYEXxlR10ASKj2dnYxl0v7eLXa/fR3ReOXs8NZXHjhafx3jNrNN4kGW1+dQkXzC6PThT6/lNbueevLgy4qvhQQEnS6ekL89KORu5fuZdnthx5y88vm1fBjReeRkVkuReRTPfRZTNZs+cEzsEL2xp5ecdRLk2DLm8FlCSFjp4+VtUfZ/nGgzy16RAtXX1vecy8qiI+ffFszphWEkCFIsmrrryQty+oim4t8+WHXue3t1yW8kt8KaAkEMfautm4v5l1e5t4ZedR1jc00ds/8uZrS08r59oltSyaVqLuPJGT+Mj5dazefZyOnn4OtXTx2XvW8ODnLqEgNxR0aROmgJKEcc5xoqOXvcc72HmkjR2NbWw/3Mbmgy3sH+OmwsriXC6eW8EVZ1RTO6VgkioWSV0VxXnceuUCvveHLYQdbNjXzN89vJ4f37iUrKzU/MVOASUT4pzjeHsPh1u6OdzaxZGWLg41d3OopYtDzZ0cbO6i4XgH7T2x71Uzs7yAs2eUcfHcCuZXF6u1JDJOS+qm8BeXzuauyG0Yyzce4jtTNvPVaxalZEgpoOSkmjt62Xm0jb3HOthzrIM9x9s50OSFz8HmLnp8s+rGKydkzKooYk5lEYumlbCotlRbsYvEwbsXT2N/UxdPbfIWkP3pi7t4Y38LP/zoOUxPsd4IBVSGc85xrL2HbYdb2X64je1HvO87G9s52tZ9yq+fn5NFVXEetVMKmBH5qisvYEZ5gTYQFEmQT188iyMtXbzW0ATAK/XHeO+/vMC3rz+T68+ZkTKtKXNu5IHpZLVs2TK3Zs2aoMtIKc45Wrr6aDjewd7I167GdnY2euNCTR0T20umMDdEeWEu5UW5lBfkUF6Uy9SiXKYW5jK1OJeqkjxK8rLVVScSgL5wmEfW7eex9fvxf8zXlRfwsWUzuWFZHbVlydGiMrO1zrllb7meyIAys6uBHwEh4GfOue8O+3kecA9wPnAM+Jhzbvdor5nMAeWco7ff0dsfpq/f0Rv2voedoz/sffczDDMwgywzsmzw3DAckcc76HeOvn7vdXr6w3T3hunq66ert5/27j5au/po6+6jqaOXEx09HG/v4WhbN4dbujnU3EVnb+xjQQNyQsa0sgJqS/OpLs2jpjSf6pI8KorymFqUm9Kzg0QyxdZDrfz7czs40jq0RyTL4PSaEpbUlbGkbgoLqoupjvwbL8qb3M61SQ8oMwsB24B3A/uA1cCNzrk3fY/578AS59zNZvZx4M+ccx8b7XVPJaD+8MZBHlzdADDw0Y9z3vHAn0PYOcJhLxDCYRf93hd2Q0Knrz9MTySMBr9SqzU6IC87K9r1VldeyIxyryuuqjgvZboCROTkOnr6eGTdfp7bdoT27rF/WS3MDVGYmx35HiI3O4vsLCM75H0f/GXayDL4/g1LqC6Z+D1XJwuoRMbkhcAO51x9pIAHgOuBN32PuR74h8jxr4Efm5m5BKVmw/FOnt3amIiXTnq52VlUl+QxrTSf2rJ8ppV5gTSzvJCK4lyy1A0nkraK87K5+R3z+KvL5vBK/TH+uOkQG/c3c7IP2o6efjrGMQO3u3fiE6ZGk8iAmgE0+M73ARed7DHOuT4zawYqgKP+B5nZ54DPRU7bzGwyluytHF5Hkhuz3u2TVMg4pNqfMajmyZBq9UKG13za9075JWaNdDElZvE55+4A7pjM9zSzNSM1OZNVqtULqnmypFrNqVYvqOZESeQ83/3ATN95XeTaiI8xs2ygDG+yhIiIZLhEBtRqYIGZzTGzXODjwOPDHvM48OeR4xuAPyVq/ElERFJLwrr4ImNKtwBP4U0zv8s5t8nMvg2scc49DtwJ3GtmO4DjeCGWLCa1SzEOUq1eUM2TJdVqTrV6QTUnRMrdqCsiIplBa82IiEhSUkCJiEhSUkD5mNlMM3vWzN40s01mdmvQNY3FzPLN7FUzez1S87eCrilWZhYys9fM7HdB1xILM9ttZhvNbL2ZJed6Wz5mNsXMfm1mW8xss5ldEnRNozGzMyJ/tgNfLWb2t0HXNRYz+1Lk394bZvYrM0vqbWzN7NZIrZuS/c9XY1A+ZlYL1Drn1plZCbAW+KB/eaZkY95KrEXOuTYzywFeAm51zq0MuLQxmdmXgWVAqXPu/UHXMxYz2w0sc86lxA2ZZvYL4EXn3M8iM2kLnXNNQdcVi8hSafuBi5xze4Ku52TMbAbev7nFzrlOM3sIWO6cuzvYykZmZmcBD+Ct9NMD/AG42Tm3I9DCTkItKB/n3EHn3LrIcSuwGW+1i6TlPG2R05zIV9L/1mFmdcC1wM+CriUdmVkZ8Ha8mbI453pSJZwirgR2JnM4+WQDBZF7OQuBAwHXM5pFwCrnXIdzrg94HvhQwDWdlALqJMxsNnAesCrYSsYW6SpbDxwB/ss5l/Q1A/8C/E8gMYt4JYYD/mhmayPLbyWzOUAj8PNIN+rPzKwo6KLG4ePAr4IuYizOuf3APwF7gYNAs3Puj8FWNao3gLeZWYWZFQLvY+iCCklFATUCMysGfgP8rXOuJeh6xuKc63fOnYu3WseFkWZ80jKz9wNHnHNrg65lnC53zi0FrgG+YGZvD7qgUWQDS4H/cM6dB7QDtwVbUmwi3ZHXAQ8HXctYzKwcb9HrOcB0oMjMPhVsVSfnnNsMfA/4I1733npg/HvxTBIF1DCRcZzfAPc75x4Jup7xiHThPAtcHXQtY7gMuC4ypvMAcIWZ3RdsSWOL/LaMc+4I8CheP36y2gfs87Wmf40XWKngGmCdc+5w0IXE4Cpgl3Ou0TnXCzwCXBpwTaNyzt3pnDvfOfd24ATetkhJSQHlE5lwcCew2Tn3z0HXEwszqzKzKZHjArz9t7YEW9XonHNfdc7VOedm43Xl/Mk5l7S/dQKYWVFk4gyRrrL34HWXJCXn3CGgwczOiFy6kqFb3SSzG0mB7r2IvcDFZlYY+fy4Em/sOmmZWXXk+2l440+/DLaik0uJ1cwn0WXAp4GNkTEdgK8555YHWNNYaoFfRGY9ZQEPOedSYtp2iqkBHo1sX58N/NI594dgSxrTF4H7I11m9cBfBlzPmCLh/27g80HXEgvn3Coz+zWwDugDXiP5lxD6jZlVAL3AF5J58oymmYuISFJSF5+IiCQlBZSIiCQlBZSIiCQlBZSIiCQlBZSIiCQlBZRkJDNzZvZD3/nfm9k/xOm17zazG+LxWmO8z0ciq5Q/O+x6lpn9a2TF6o1mttrM5kR+9rVE1yUSLwooyVTdwIfMrDLoQvwiC47G6ibgs865dw27/jG8ZXeWOOfOBv4MGLjXRQElKUMBJZmqD++Gyi8N/8HwFpCZtUW+v9PMnjez35pZvZl918w+GdmPa6OZzfO9zFVmtsbMtkXWHhxY1PcHkRbNBjP7vO91XzSzxxlhtQczuzHy+m+Y2fci174JXA7caWY/GPaUWuCgcy4M4Jzb55w7YWbfxVt1e72Z3R95nU9F6l9vZj+J3PCNmbWZ2f+N7Bn0jJlVRa7/jXn7pW0wswcm8OcuEjMFlGSy24FPRramiNU5wM142xZ8GjjdOXch3rYhX/Q9bjbeWn3XAv9p3iZ2N+Gtdn0BcAHw2YGuN7x18m51zp3ufzMzm463uOcVwLnABWb2Qefct4E1wCedc/9jWI0PAR+IhM4Pzew8AOfcbUCnc+5c59wnzWwRXmvrsshiw/3AJyOvUQSscc6dibclw/8XuX4bcJ5zbknkz0EkYRRQkrEiK9XfA/zNOJ62OrJvWDewE29VaICNeKE04CHnXNg5tx1vmaGFeOv3fSayjNYqoAJYEHn8q865XSO83wXAc5HFSPuA+/H2eRrtv2sfcAbwVbztTJ4xsytHeOiVwPnA6khNVwJzIz8LAw9Gju/Da60BbMBbPulTeK1QkYTRWnyS6f4Fbx21n/uu9RH55c3MsoBc38+6fcdh33mYof+ehq8h5gADvuice8r/AzN7J952GHETCdDfA783s8PAB4Fnhj3MgF84574ay0tGvl+LF5AfAL5uZmdHglMk7tSCkozmnDuO1yV2k+/ybryWBXj7EuVM4KU/EplNNw+vVbIVeAr468iWLpjZ6TFsIvgq8A4zq4yMD92I1+V2Uma2NNI1OBCwS4CBnWl7B94fL7Bu8K1uPdXMZkV+lgUMjMN9Angp8loznXPPAl8ByoDiMf8kRCZILSgR+CFwi+/8p8Bvzex1vE3dJtK62YsXLqXAzc65LjP7GV434LrI1gyNeC2bk3LOHTSz2/D2+TLgSefcb8d472rgp2aWFzl/Ffhx5PgOYIOZrYuMQ30Db5fgLCKrW+OFWTve5pffwNup+WNACLgvMmZnwL8m80rYkvq0mrmIvIWZtTnn1DqSQKmLT0REkpJaUCIikpTUghIRkaSkgBIRkaSkgBIRkaSkgBIRkaSkgBIRkaT0/wCDJ3GLrvnshwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = APGCN(dataset,10,prop_penalty=0.002)\n",
    "\n",
    "results = run(\n",
    "    dataset,\n",
    "    model,\n",
    "    #seeds=test_seeds if config['test'] else val_seeds,\n",
    "    seeds= quick_seeds,\n",
    "    lr=config['lr'],\n",
    "    weight_decay=config['weight_decay'],\n",
    "    test=config['test'],\n",
    "    num_development=config['num_development'],\n",
    "    device=device\n",
    "    )\n",
    "\n",
    "save_obj(results,'results_' + config['dataset_name'])\n",
    "report = summary(results)\n",
    "\n",
    "print(\"FINAL\\n\"\n",
    "      \"Train Accuracy: {:.2f} ± {:.2f}%\\n\"\n",
    "      \"Stopping Accuracy: {:.2f} ± {:.2f}%\\n\"\n",
    "      \"Test     Accuracy: {:.2f} ± {:.2f}%\\n\"\n",
    "      \"Steps: {:.2f} ± {:.2f}\\n\" \n",
    "      \"Epochs:  {:.2f} ± {:.2f}\\n\"\n",
    "      \"Runtime: {:.4f} ± {:.4f}\\n\"\n",
    "      .format(\n",
    "          report['train_acc'] * 100,\n",
    "          report['train_acc_ci'] * 100,\n",
    "          report['val_acc'] * 100,\n",
    "          report['val_acc_ci'] * 100,\n",
    "          report['test_acc']*100,\n",
    "          report['test_acc_ci']*100,\n",
    "          report['steps'],\n",
    "          report['steps_ci'],\n",
    "          report['epoch'],\n",
    "          report['epoch_ci'],\n",
    "          report['runtime'],\n",
    "          report['runtime_ci']))\n",
    "\n",
    "plot_density(results)\n",
    "\n",
    "del model, dataset\n",
    "torch.cuda.empty_cache()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}