AttentionSleep_Model_Sleep_TL.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "AttentionSleep Model - Sleep TL.ipynb",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyMG3iD2Vq5Ekwm3Wjxl0TiH",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Div12345/SleepStaging-TransferLearning/blob/main/AttentionSleep_Model_Sleep_TL.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L9sQe2m-bYJE"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m5JTtD-_8Dy0",
        "cellView": "form"
      },
      "source": [
        "#@title Dependencies Install\n",
        "!pip install wandb\n",
        "!pip install git+https://github.com/sylvchev/beetl-competition\n",
        "!pip install moabb\n",
        "!pip install braindecode\n",
        "!pip install git+https://github.com/pyRiemann/pyRiemann\n",
        "!pip install matplotlib\n",
        "!pip install https://github.com/ufoym/imbalanced-dataset-sampler/archive/master.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uuMPQYfn8LbD",
        "cellView": "form"
      },
      "source": [
        "#@title Imports\n",
        "from mne import get_config, set_config\n",
        "import os.path as osp\n",
        "import os\n",
        "from beetl.task_datasets import BeetlSleepLeaderboard, BeetlSleepSource\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import mne\n",
        "\n",
        "import braindecode\n",
        "from braindecode import EEGClassifier\n",
        "from braindecode.util import np_to_var, set_random_seeds\n",
        "from braindecode.models import SleepStagerChambon2018\n",
        "from braindecode.datautil.preprocess import preprocess, Preprocessor, zscore\n",
        "from braindecode.samplers.ssl import RelativePositioningSampler\n",
        "#from braindecode.datautil import create_from_X_y\n",
        "from braindecode.datasets import BaseDataset\n",
        "\n",
        "import sklearn\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.metrics import confusion_matrix, classification_report, balanced_accuracy_score, cohen_kappa_score\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.lines import Line2D\n",
        "\n",
        "import skorch\n",
        "import skorch.dataset\n",
        "from skorch.callbacks import EarlyStopping, Checkpoint, EpochScoring, WandbLogger, TrainEndCheckpoint\n",
        "from skorch.dataset import Dataset\n",
        "from skorch.helper import predefined_split\n",
        "from skorch.classifier import NeuralNetClassifier as NNClassifier\n",
        "\n",
        "import time\n",
        "import math\n",
        "import copy\n",
        "from copy import deepcopy\n",
        "\n",
        "import pickle\n",
        "import torch\n",
        "from torch import nn\n",
        "from torch.utils.data import DataLoader\n",
        "import torch.nn.functional as F\n",
        "\n",
        "from torchsampler import ImbalancedDatasetSampler as IDS\n",
        "\n",
        "import wandb\n",
        "\n",
        "import joblib"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JsEpic6LddRa",
        "outputId": "63440407-60c6-4d04-94bf-fc384c633ae0"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6H7g78UA8OBg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4070091c-8a9c-405c-8c1f-ee16ea3f145b"
      },
      "source": [
        "# Optional wandb for model logging\n",
        "!wandb login\n",
        "# Specific to user - Put it here for convinience"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_h42a0Z8SlJ",
        "outputId": "21d2d8ff-521a-42a0-87c9-863eefd148a4"
      },
      "source": [
        "mne.set_log_level(False) # Equivalent to WARNING\n",
        "path = \"/content/drive/MyDrive/mne_data\"\n",
        "set_config(\"MNE_DATA\", path)\n",
        "set_config(\"MNE_DATASETS_BEETLSLEEPLEADERBOARD_PATH\",path)\n",
        "set_config(\"MNE_DATASETS_BEETLSLEEPSOURCE_PATH\",path)\n",
        "get_config()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-4-d9ce88047e42>:4: RuntimeWarning: Setting non-standard config type: \"MNE_DATASETS_BEETLSLEEPLEADERBOARD_PATH\"\n",
            "  set_config(\"MNE_DATASETS_BEETLSLEEPLEADERBOARD_PATH\",path)\n",
            "<ipython-input-4-d9ce88047e42>:5: RuntimeWarning: Setting non-standard config type: \"MNE_DATASETS_BEETLSLEEPSOURCE_PATH\"\n",
            "  set_config(\"MNE_DATASETS_BEETLSLEEPSOURCE_PATH\",path)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'MNE_DATA': '/content/drive/MyDrive/mne_data',\n",
              " 'MNE_DATASETS_BEETLSLEEPLEADERBOARD_PATH': '/content/drive/MyDrive/mne_data',\n",
              " 'MNE_DATASETS_BEETLSLEEPSOURCE_PATH': '/content/drive/MyDrive/mne_data'}"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E_fQU4bJ8TF2",
        "outputId": "7934f309-def9-48a6-cf1b-301d8e06677b"
      },
      "source": [
        "cuda = torch.cuda.is_available()  # check if GPU is available\n",
        "device = 'cuda' if cuda else 'cpu'\n",
        "print(device)\n",
        "if cuda:\n",
        "    torch.backends.cudnn.benchmark = True\n",
        "# Set random seed to be able to reproduce results\n",
        "set_random_seeds(seed=87, cuda=cuda)\n",
        "random_state = 87\n",
        "\n",
        "!nvidia-smi -L\n",
        "# !nvidia-smi -q\n",
        "\n",
        "# gpu = cuda.get_current_device()\n",
        "# print(\"maxThreadsPerBlock = %s\" % str(gpu.MAX_THREADS_PER_BLOCK))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n",
            "GPU 0: Tesla P100-PCIE-16GB (UUID: GPU-c3e06580-ab8e-2ffd-1dc1-f3ea7be7b9c2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QLNWv8j48Vhd",
        "cellView": "form"
      },
      "source": [
        "#@title Helper Functions\n",
        "\n",
        "def label_count(y_train):\n",
        "  labels= np.unique(y_train)\n",
        "  labelsize=labels.shape[0]\n",
        "  #print('labelsize:',labelsize)\n",
        "  label_count = np.zeros(labelsize).astype(int)\n",
        "  for i in range(labelsize):\n",
        "      # tempy = ys1[ys1==labels[i]]\n",
        "      label_count[i]=y_train[y_train==labels[i]].shape[0]\n",
        "  maxsize = label_count.max()\n",
        "  print(label_count)\n",
        "  return np_to_var(label_count)\n",
        "\n",
        "def label_viz(y):\n",
        "  # Another Nice func with viz\n",
        "  classes_mapping = {0: 'W', 1: 'S1', 2: 'S2', 3: 'S3', 4: 'S4', 5:'REM'}\n",
        "  # This might be a time consuming method though\n",
        "  y_train = pd.Series([y for _, y in train_ds]).map(classes_mapping)\n",
        "  ax = y_train.value_counts().plot(kind='barh')\n",
        "  ax.set_xlabel('Number of training examples');\n",
        "  ax.set_ylabel('Sleep stage');\n",
        "\n",
        "# For trained Skorch model\n",
        "def training_viz(clf):\n",
        "  # For Trained Skorch Classifier\n",
        "  df = pd.DataFrame(clf.history.to_list())\n",
        "  df[['train_mis_clf', 'valid_mis_clf']] = 100 - df[\n",
        "      ['train_bacc', 'valid_bacc']] * 100\n",
        "\n",
        "  # get percent of misclass for better visual comparison to loss\n",
        "  plt.style.use('seaborn-talk')\n",
        "  fig, ax1 = plt.subplots(figsize=(20, 7))\n",
        "  df.loc[:, ['train_loss', 'valid_loss']].plot(\n",
        "      ax=ax1, style=['-', ':'], marker='o', color='tab:blue', legend=False,\n",
        "      fontsize=14)\n",
        "\n",
        "  ax1.tick_params(axis='y', labelcolor='tab:blue', labelsize=14)\n",
        "  ax1.set_ylabel(\"Loss\", color='tab:blue', fontsize=14)\n",
        "\n",
        "  ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis\n",
        "\n",
        "  df.loc[:, ['train_mis_clf', 'valid_mis_clf']].plot(\n",
        "      ax=ax2, style=['-', ':'], marker='o', color='tab:red', legend=False)\n",
        "  ax2.tick_params(axis='y', labelcolor='tab:red', labelsize=14)\n",
        "  ax2.set_ylabel('Balanced misclassification rate [%]', color='tab:red',\n",
        "                fontsize=14)\n",
        "  ax2.set_ylim(ax2.get_ylim()[0], 85)  # make some room for legend\n",
        "  ax1.set_xlabel('Epoch', fontsize=14)\n",
        "\n",
        "  # where some data has already been plotted to ax\n",
        "  handles = []\n",
        "  handles.append(\n",
        "      Line2D([0], [0], color='black', linewidth=1, linestyle='-', label='Train'))\n",
        "  handles.append(\n",
        "      Line2D([0], [0], color='black', linewidth=1, linestyle=':', label='Valid'))\n",
        "  plt.legend(handles, [h.get_label() for h in handles], fontsize=14)\n",
        "  plt.tight_layout()\n",
        "\n",
        "# Scaling the data\n",
        "class TrainObject(object):\n",
        "    # Scaling the data\n",
        "    def __init__(self, X, y, nps= False):\n",
        "        assert len(X) == len(y)\n",
        "        # mean = np.mean(X, axis=2, keepdims=True)\n",
        "        # # Here normalise across the window, when channel size is not large enough\n",
        "        # # In motor imagery kit, we put axis = 1, across channel as an example\n",
        "        # std = np.std(X, axis=2, keepdims=True)\n",
        "        # X = (X - mean) / std\n",
        "        X = zscore(X)\n",
        "        # we scale it to 1000 as a better training scale of the shallow CNN\n",
        "        # according to the orignal work of the paper referenced above\n",
        "        if (nps == False):\n",
        "          self.X = np_to_var(X.astype(np.float32)*1e3).to(device)\n",
        "          self.y = np_to_var(y.astype(np.int8)).to(device)\n",
        "        else:\n",
        "          self.X = X.astype(np.float32)*1e3\n",
        "          self.y = y.astype(np.int8)\n",
        "\n",
        "def predict_leaderboard_unlabelled(clf, save_fname, x_test_data = None, emb = False,nps=False):\n",
        "  # Test Data - 6 to 17 - 12 subjects \n",
        "  if x_test_data is None:\n",
        "    _, _, X_test, _ = dsl.get_data(subjects=range(6, 18)) \n",
        "    print(\"Sleep leaderboard - Test Data : There are {} trials with {} electrodes and {} time samples\".format(*X_test.shape))\n",
        "\n",
        "    # print(X_test.shape[0])\n",
        "\n",
        "    x_test_mean = TrainObject(X_test, y = np.zeros(X_test.shape[0]),nps=nps)\n",
        "    if (nps==False):\n",
        "      # means torch must've been the input to the classifier\n",
        "      x_test_data = Dataset(x_test_mean.X,x_test_mean.y)\n",
        "      # Maybe put a tqdm bar?\n",
        "      y_pred = clf.predict(x_test_data)\n",
        "    else:\n",
        "      x_test_data = x_test_mean \n",
        "      y_pred = clf.predict(x_test_data.X)\n",
        "  else:\n",
        "    y_pred = clf.predict(x_test_data)\n",
        "  print(x_test_data.shape)\n",
        "\n",
        "  print(\"Checking if all classes have been predicted\")\n",
        "  print(np.unique(y_pred))\n",
        "  #@markdown Set the save_path for saving the prediction\n",
        "  save_path = \"/content/drive/MyDrive/mne_data/NeuroIPS_Hack_Sleep/predict/\" #@param\n",
        "  np.savetxt(save_path+save_fname+\".txt\",y_pred,delimiter=',',fmt=\"%d\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kHWXsvutVDsq",
        "cellView": "form"
      },
      "source": [
        "#@title Helper Functions for loading the Data\n",
        "def get_trainB():\n",
        "  start = time.time()\n",
        "  dss = BeetlSleepSource()\n",
        "  \n",
        "  start = time.time()\n",
        "  X_train, y_train, _ = dss.get_data()\n",
        "  end = time.time()\n",
        "\n",
        "  print(f\"Train data load time {(end-start)/60} min\")\n",
        "  print(X_train.shape,y_train.shape)\n",
        "  label_count(y_train)\n",
        "\n",
        "  # z-scoring or normalizing\n",
        "  \n",
        "  # start = time.time()\n",
        "  trainX = TrainObject(X_train, y = y_train, nps = False)\n",
        "  end = time.time()\n",
        "  print(f\"Z-scoring time {end-start}\")\n",
        "  \n",
        "  start = time.time()\n",
        "  # trainB = Dataset(trainX)\n",
        "  trainB = Dataset(trainX.X,trainX.y)\n",
        "  end = time.time()\n",
        "  print(f\"Torch dataset time {end-start} sec = {(end-start)/60} min\")\n",
        "  # 10 sec + 7 min\n",
        "  return trainB\n",
        "\n",
        "def get_valB():\n",
        "  # valB\n",
        "  # Labelled Leaderboard Data\n",
        "  start = time.time()\n",
        "  dsl = BeetlSleepLeaderboard()\n",
        "  # dsl.get_data()\n",
        "\n",
        "  # Validation Data - 5? 6? subjects from the test group - Competition says 5, looks like 6\n",
        "\n",
        "  X_target, y_target, _, _ = dsl.get_data(subjects=range(0,6))\n",
        "\n",
        "  end = time.time()\n",
        "  print(f\"leaderboard labelled load time = {(end-start)/60} min\")\n",
        "\n",
        "  print(X_target.shape,y_target.shape)\n",
        "  label_count(y_target)\n",
        "\n",
        "  # z-scoring or normalizing\n",
        "  start = time.time()\n",
        "  valX = TrainObject(X_target, y = y_target, nps = False)\n",
        "  end = time.time()\n",
        "  print(f\"Z-scoring time {end-start} sec\")\n",
        "\n",
        "  \n",
        "  # for braindecode\n",
        "  start = time.time()\n",
        "  valB = Dataset(valX.X,valX.y)\n",
        "  end = time.time()\n",
        "  print(f\"Torch dataset time {end-start} sec = {(end-start)/60} min\")\n",
        "  # 3 sec + 1.5 min\n",
        "  return valB\n",
        "\n",
        "def get_testB():\n",
        "  dsl = BeetlSleepLeaderboard()\n",
        "  _, _, X_test, _ = dsl.get_data(subjects=range(6, 18)) \n",
        "  print(\"Sleep leaderboard - Test Data : There are {} trials with {} electrodes and {} time samples\".format(*X_test.shape))\n",
        "  # print(X_test.shape[0])\n",
        "  # x_test_mean = TrainObject(X_test, y = np.zeros(X_test.shape[0]),nps=nps)\n",
        "\n",
        "  # z-scoring or normalizing\n",
        "  start = time.time()\n",
        "  testX = TrainObject(X_test, y = np.zeros(X_test.shape[0]), nps = False)\n",
        "  end = time.time()\n",
        "  print(f\"Z-scoring time {end-start} sec\")\n",
        "  \n",
        "  # for braindecode\n",
        "  start = time.time()\n",
        "  testB = Dataset(testX.X,testX.y)\n",
        "  end = time.time()\n",
        "  print(f\"Torch dataset time {end-start} sec = {(end-start)/60} min\")\n",
        "  return testB\n",
        "\n",
        "def get_valC():\n",
        "  # Phase 2 Target Data\n",
        "  # Need to download by myself first\n",
        "  target_savebase = '/content/drive/MyDrive/mne_data/MNE-beetlsleeptest-data/sleep_target/'\n",
        "  X_sleep_target = []\n",
        "  y_sleep_target = []\n",
        "  #from s0-s4 in final set\n",
        "  start = time.time()\n",
        "  for subj in range(0, 5):\n",
        "      for session in range(1, 3):\n",
        "          # \"testing_s{}r{}X.npy\", replacing \"leaderboard_s{}r{}X.npy\" before\n",
        "          with open(target_savebase + \"testing_s{}r{}X.npy\".format(subj, session), 'rb') as f:\n",
        "            X_sleep_target.append(pickle.load(f))\n",
        "          with open(target_savebase + \"testing_s{}r{}y.npy\".format(subj, session), 'rb') as g:\n",
        "            y_sleep_target.append(pickle.load(g))\n",
        "  X_sleep_target = np.concatenate(X_sleep_target)\n",
        "  y_sleep_target = np.concatenate(y_sleep_target)\n",
        "\n",
        "\n",
        "  end = time.time()\n",
        "  print(f\"phase 2 labelled load time = {(end-start)/60} min\")\n",
        "\n",
        "  print(\"There are {} trials with {} electrodes and {} time samples\".format(*X_sleep_target.shape))\n",
        "  print(X_sleep_target.shape, y_sleep_target.shape)\n",
        "  label_count(y_sleep_target)\n",
        "\n",
        "  # package in torch dataset with mean normalizing\n",
        "  # z-scoring or normalizing\n",
        "  start = time.time()\n",
        "  valX = TrainObject(X_sleep_target, y = y_sleep_target, nps = False)\n",
        "  end = time.time()\n",
        "  print(f\"Z-scoring time {end-start} sec\")\n",
        "\n",
        "  \n",
        "  # for braindecode\n",
        "  start = time.time()\n",
        "  valC = Dataset(valX.X,valX.y)\n",
        "  end = time.time()\n",
        "  print(f\"Torch dataset time {end-start} sec = {(end-start)/60} min\")\n",
        "  # \n",
        "  return valC\n",
        "\n",
        "def get_testC():\n",
        "  # Phase 2 Test Data - Need to download by myself first\n",
        "  test_savebase = '/content/drive/MyDrive/mne_data/MNE-beetlsleeptest-data/testing/'\n",
        "  X_sleep_test = []\n",
        "  start = time.time()\n",
        "\n",
        "  #starts from s5 in final set\n",
        "  for subj in range(5, 14):\n",
        "      for session in range(1, 3):\n",
        "          # \"testing_s{}r{}X.npy\", replacing \"leaderboard_s{}r{}X.npy\" before\n",
        "          with open(test_savebase + \"testing_s{}r{}X.npy\".format(subj, session), 'rb') as f:\n",
        "              X_sleep_test.append(pickle.load(f))\n",
        "  X_sleep_test = np.concatenate(X_sleep_test)\n",
        "  end = time.time()\n",
        "  print(f\"phase 2 final test set load time = {(end-start)/60} min\")\n",
        "  print (\"There are {} trials with {} electrodes and {} time samples\".format(*X_sleep_test.shape))\n",
        "\n",
        "  # package in torch dataset with mean normalizing\n",
        "  # z-scoring or normalizing\n",
        "  start = time.time()\n",
        "  testX = TrainObject(X_sleep_test, y = np.zeros(X_sleep_test.shape[0]), nps = False)\n",
        "  end = time.time()\n",
        "  print(f\"Z-scoring time {end-start} sec\")\n",
        "  \n",
        "  # for braindecode\n",
        "  start = time.time()\n",
        "  testC = Dataset(testX.X,testX.y)\n",
        "  end = time.time()\n",
        "  print(f\"Torch dataset time {end-start} sec = {(end-start)/60} min\")\n",
        "  return testC"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VFdTDRSBXXXM"
      },
      "source": [
        "from time import time as t\n",
        "def load_obj(path,name):\n",
        "  target = path + name + '.pkl'\n",
        "  with open(target, 'rb') as f:\n",
        "    unpickler = pickle.Unpickler(f)\n",
        "    a = unpickler.load()\n",
        "    return a"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kRK7hmlqXtJB",
        "outputId": "9143d181-759e-44fc-89a8-229bdbdba94c"
      },
      "source": [
        "# Loading the data as Torch datasets with z-scoring\n",
        "trainB = get_trainB()\n",
        "valB = get_valB()\n",
        "testB = get_testB()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train data load time 2.091049063205719 min\n",
            "(90545, 2, 3000) (90545,)\n",
            "[24043  7941 35983  5247  3057 14274]\n",
            "Z-scoring time 142.37633633613586\n",
            "Torch dataset time 0.0035288333892822266 sec = 5.8813889821370444e-05 min\n",
            "leaderboard labelled load time = 0.3546595533688863 min\n",
            "(15442, 2, 3000) (15442,)\n",
            "[6010 1672 5035  704  414 1607]\n",
            "Z-scoring time 1.0841395854949951 sec\n",
            "Torch dataset time 9.655952453613281e-05 sec = 1.6093254089355469e-06 min\n",
            "Sleep leaderboard - Test Data : There are 25748 trials with 2 electrodes and 3000 time samples\n",
            "Z-scoring time 1.6269407272338867 sec\n",
            "Torch dataset time 7.843971252441406e-05 sec = 1.3073285420735677e-06 min\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eV32LyTYm4Sr",
        "outputId": "d0ac10da-276c-4664-caf6-d2cee351441a"
      },
      "source": [
        "print(trainB)\n",
        "print(valB)\n",
        "print(testB)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<skorch.dataset.Dataset object at 0x7f408e16acd0>\n",
            "<skorch.dataset.Dataset object at 0x7f40958a1790>\n",
            "<skorch.dataset.Dataset object at 0x7f41141f1050>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "suXuaOiVFnXb"
      },
      "source": [
        "# AttnSleep"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xYDHjZIXYEvL",
        "cellView": "form"
      },
      "source": [
        "#@title Utilities Functions\n",
        "# calc_class_weight\n",
        "def calc_class_weight(labels_count):\n",
        "    total = np.sum(labels_count)\n",
        "    class_weight = dict()\n",
        "    num_classes = len(labels_count)\n",
        "\n",
        "    factor = 1 / num_classes\n",
        "    # adding a factor for S4 as 1.5\n",
        "    # original work was AASM only till N3 = S3+S4\n",
        "    mu = [factor * 1.5, factor * 2, factor * 1.5, factor,factor*1.5, factor * 1.5]\n",
        "\n",
        "    for key in range(num_classes):\n",
        "        score = math.log(mu[key] * total / float(labels_count[key]))\n",
        "        class_weight[key] = score if score > 1.0 else 1.0\n",
        "        class_weight[key] = round(class_weight[key] * factor, 2)\n",
        "\n",
        "    class_weight = [class_weight[i] for i in range(num_classes)]\n",
        "\n",
        "    return class_weight"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v7svPvZ1OX-W",
        "cellView": "form"
      },
      "source": [
        "#@title AttnSleep helper modules\n",
        "\n",
        "class SELayer(nn.Module):\n",
        "    def __init__(self, channel, reduction=16):\n",
        "        super(SELayer, self).__init__()\n",
        "        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(channel, channel // reduction, bias=False),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.Linear(channel // reduction, channel, bias=False),\n",
        "            nn.Sigmoid()\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        b, c, _ = x.size()\n",
        "        y = self.avg_pool(x).view(b, c)\n",
        "        y = self.fc(y).view(b, c, 1)\n",
        "        return x * y.expand_as(x)\n",
        "\n",
        "\n",
        "class SEBasicBlock(nn.Module):\n",
        "    expansion = 1\n",
        "\n",
        "    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n",
        "                 base_width=64, dilation=1, norm_layer=None,\n",
        "                 *, reduction=16):\n",
        "        super(SEBasicBlock, self).__init__()\n",
        "        self.conv1 = nn.Conv1d(inplanes, planes, stride)\n",
        "        self.bn1 = nn.BatchNorm1d(planes)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.conv2 = nn.Conv1d(planes, planes, 1)\n",
        "        self.bn2 = nn.BatchNorm1d(planes)\n",
        "        self.se = SELayer(planes, reduction)\n",
        "        self.downsample = downsample\n",
        "        self.stride = stride\n",
        "        \n",
        "\n",
        "    def forward(self, x):\n",
        "        residual = x\n",
        "        out = self.conv1(x)\n",
        "        out = self.bn1(out)\n",
        "        out = self.relu(out)\n",
        "\n",
        "        out = self.conv2(out)\n",
        "        out = self.bn2(out)\n",
        "        out = self.se(out)\n",
        "\n",
        "        if self.downsample is not None:\n",
        "            residual = self.downsample(x)\n",
        "\n",
        "        out += residual\n",
        "        out = self.relu(out)\n",
        "\n",
        "        return out\n",
        "\n",
        "class GELU(nn.Module):\n",
        "    # for older versions of PyTorch.  For new versions you can use nn.GELU() instead.\n",
        "    def __init__(self):\n",
        "        super(GELU, self).__init__()\n",
        "        \n",
        "    def forward(self, x):\n",
        "        x = torch.nn.functional.gelu(x)\n",
        "        return x\n",
        "        \n",
        "        \n",
        "class MRCNN(nn.Module):\n",
        "    def __init__(self, afr_reduced_cnn_size):\n",
        "        super(MRCNN, self).__init__()\n",
        "        drate = 0.5\n",
        "        self.GELU = GELU()  # for older versions of PyTorch.  For new versions use nn.GELU() instead.\n",
        "        self.features1 = nn.Sequential(\n",
        "            nn.Conv1d(1, 64, kernel_size=50, stride=6, bias=False, padding=24),\n",
        "            nn.BatchNorm1d(64),\n",
        "            self.GELU,\n",
        "            nn.MaxPool1d(kernel_size=8, stride=2, padding=4),\n",
        "            nn.Dropout(drate),\n",
        "\n",
        "            nn.Conv1d(64, 128, kernel_size=8, stride=1, bias=False, padding=4),\n",
        "            nn.BatchNorm1d(128),\n",
        "            self.GELU,\n",
        "\n",
        "            nn.Conv1d(128, 128, kernel_size=8, stride=1, bias=False, padding=4),\n",
        "            nn.BatchNorm1d(128),\n",
        "            self.GELU,\n",
        "\n",
        "            nn.MaxPool1d(kernel_size=4, stride=4, padding=2)\n",
        "        )\n",
        "\n",
        "        self.features2 = nn.Sequential(\n",
        "            nn.Conv1d(1, 64, kernel_size=400, stride=50, bias=False, padding=200),\n",
        "            nn.BatchNorm1d(64),\n",
        "            self.GELU,\n",
        "            nn.MaxPool1d(kernel_size=4, stride=2, padding=2),\n",
        "            nn.Dropout(drate),\n",
        "\n",
        "            nn.Conv1d(64, 128, kernel_size=7, stride=1, bias=False, padding=3),\n",
        "            nn.BatchNorm1d(128),\n",
        "            self.GELU,\n",
        "\n",
        "            nn.Conv1d(128, 128, kernel_size=7, stride=1, bias=False, padding=3),\n",
        "            nn.BatchNorm1d(128),\n",
        "            self.GELU,\n",
        "\n",
        "            nn.MaxPool1d(kernel_size=2, stride=2, padding=1)\n",
        "        )\n",
        "        self.dropout = nn.Dropout(drate)\n",
        "        self.inplanes = 128\n",
        "        self.AFR = self._make_layer(SEBasicBlock, afr_reduced_cnn_size, 1)\n",
        "\n",
        "    def _make_layer(self, block, planes, blocks, stride=1):  # makes residual SE block\n",
        "        downsample = None\n",
        "        if stride != 1 or self.inplanes != planes * block.expansion:\n",
        "            downsample = nn.Sequential(\n",
        "                nn.Conv1d(self.inplanes, planes * block.expansion,\n",
        "                          kernel_size=1, stride=stride, bias=False),\n",
        "                nn.BatchNorm1d(planes * block.expansion),\n",
        "            )\n",
        "\n",
        "        layers = []\n",
        "        layers.append(block(self.inplanes, planes, stride, downsample))\n",
        "        self.inplanes = planes * block.expansion\n",
        "        for i in range(1, blocks):\n",
        "            layers.append(block(self.inplanes, planes))\n",
        "\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x1 = self.features1(x)\n",
        "        x2 = self.features2(x)\n",
        "        x_concat = torch.cat((x1, x2), dim=2)\n",
        "        x_concat = self.dropout(x_concat)\n",
        "        x_concat = self.AFR(x_concat)\n",
        "        return x_concat\n",
        "\n",
        "##########################################################################################\n",
        "\n",
        "\n",
        "def attention(query, key, value, dropout=None):\n",
        "    \"Implementation of Scaled dot product attention\"\n",
        "    d_k = query.size(-1)\n",
        "    scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n",
        "\n",
        "    p_attn = F.softmax(scores, dim=-1)\n",
        "    if dropout is not None:\n",
        "        p_attn = dropout(p_attn)\n",
        "    return torch.matmul(p_attn, value), p_attn\n",
        "\n",
        "\n",
        "class CausalConv1d(torch.nn.Conv1d):\n",
        "    def __init__(self,\n",
        "                 in_channels,\n",
        "                 out_channels,\n",
        "                 kernel_size,\n",
        "                 stride=1,\n",
        "                 dilation=1,\n",
        "                 groups=1,\n",
        "                 bias=True):\n",
        "        self.__padding = (kernel_size - 1) * dilation\n",
        "\n",
        "        super(CausalConv1d, self).__init__(\n",
        "            in_channels,\n",
        "            out_channels,\n",
        "            kernel_size=kernel_size,\n",
        "            stride=stride,\n",
        "            padding=self.__padding,\n",
        "            dilation=dilation,\n",
        "            groups=groups,\n",
        "            bias=bias)\n",
        "\n",
        "    def forward(self, input):\n",
        "        result = super(CausalConv1d, self).forward(input)\n",
        "        if self.__padding != 0:\n",
        "            return result[:, :, :-self.__padding]\n",
        "        return result\n",
        "\n",
        "class MultiHeadedAttention(nn.Module):\n",
        "    def __init__(self, h, d_model, afr_reduced_cnn_size, dropout=0.1):\n",
        "        \"Take in model size and number of heads.\"\n",
        "        super(MultiHeadedAttention, self).__init__()\n",
        "        assert d_model % h == 0\n",
        "        self.d_k = d_model // h\n",
        "        self.h = h\n",
        "\n",
        "        self.convs = clones(CausalConv1d(afr_reduced_cnn_size, afr_reduced_cnn_size, kernel_size=7, stride=1), 3)\n",
        "        self.linear = nn.Linear(d_model, d_model)\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "\n",
        "    def forward(self, query, key, value):\n",
        "        \"Implements Multi-head attention\"\n",
        "        nbatches = query.size(0)\n",
        "\n",
        "        query = query.view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n",
        "        key   = self.convs[1](key).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n",
        "        value = self.convs[2](value).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n",
        "\n",
        "        x, self.attn = attention(query, key, value, dropout=self.dropout)\n",
        "\n",
        "        x = x.transpose(1, 2).contiguous() \\\n",
        "            .view(nbatches, -1, self.h * self.d_k)\n",
        "\n",
        "        return self.linear(x)\n",
        "\n",
        "\n",
        "class LayerNorm(nn.Module):\n",
        "    \"Construct a layer normalization module.\"\n",
        "\n",
        "    def __init__(self, features, eps=1e-6):\n",
        "        super(LayerNorm, self).__init__()\n",
        "        self.a_2 = nn.Parameter(torch.ones(features))\n",
        "        self.b_2 = nn.Parameter(torch.zeros(features))\n",
        "        self.eps = eps\n",
        "\n",
        "    def forward(self, x):\n",
        "        mean = x.mean(-1, keepdim=True)\n",
        "        std = x.std(-1, keepdim=True)\n",
        "        return self.a_2 * (x - mean) / (std + self.eps) + self.b_2\n",
        "\n",
        "\n",
        "class SublayerOutput(nn.Module):\n",
        "    '''\n",
        "    A residual connection followed by a layer norm.\n",
        "    '''\n",
        "\n",
        "    def __init__(self, size, dropout):\n",
        "        super(SublayerOutput, self).__init__()\n",
        "        self.norm = LayerNorm(size)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, x, sublayer):\n",
        "        \"Apply residual connection to any sublayer with the same size.\"\n",
        "        return x + self.dropout(sublayer(self.norm(x)))\n",
        "\n",
        "\n",
        "def clones(module, N):\n",
        "    \"Produce N identical layers.\"\n",
        "    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])\n",
        "\n",
        "\n",
        "class TCE(nn.Module):\n",
        "    '''\n",
        "    Transformer Encoder\n",
        "    It is a stack of N layers.\n",
        "    '''\n",
        "\n",
        "    def __init__(self, layer, N):\n",
        "        super(TCE, self).__init__()\n",
        "        self.layers = clones(layer, N)\n",
        "        self.norm = LayerNorm(layer.size)\n",
        "\n",
        "    def forward(self, x):\n",
        "        for layer in self.layers:\n",
        "            x = layer(x)\n",
        "        return self.norm(x)\n",
        "\n",
        "\n",
        "class EncoderLayer(nn.Module):\n",
        "    '''\n",
        "    An encoder layer\n",
        "    Made up of self-attention and a feed forward layer.\n",
        "    Each of these sublayers have residual and layer norm, implemented by SublayerOutput.\n",
        "    '''\n",
        "    def __init__(self, size, self_attn, feed_forward, afr_reduced_cnn_size, dropout):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        self.self_attn = self_attn\n",
        "        self.feed_forward = feed_forward\n",
        "        self.sublayer_output = clones(SublayerOutput(size, dropout), 2)\n",
        "        self.size = size\n",
        "        self.conv = CausalConv1d(afr_reduced_cnn_size, afr_reduced_cnn_size, kernel_size=7, stride=1, dilation=1)\n",
        "\n",
        "\n",
        "    def forward(self, x_in):\n",
        "        \"Transformer Encoder\"\n",
        "        query = self.conv(x_in)\n",
        "        x = self.sublayer_output[0](query, lambda x: self.self_attn(query, x_in, x_in))  # Encoder self-attention\n",
        "        return self.sublayer_output[1](x, self.feed_forward)\n",
        "\n",
        "\n",
        "class PositionwiseFeedForward(nn.Module):\n",
        "    \"Positionwise feed-forward network.\"\n",
        "\n",
        "    def __init__(self, d_model, d_ff, dropout=0.1):\n",
        "        super(PositionwiseFeedForward, self).__init__()\n",
        "        self.w_1 = nn.Linear(d_model, d_ff)\n",
        "        self.w_2 = nn.Linear(d_ff, d_model)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"Implements FFN equation.\"\n",
        "        return self.w_2(self.dropout(F.relu(self.w_1(x))))\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BtD0xXmUzWcd",
        "cellView": "form"
      },
      "source": [
        "#@title AttnSleep Model for Physionet\n",
        "\n",
        "class AttnSleep(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(AttnSleep, self).__init__()\n",
        "\n",
        "        N = 2  # number of TCE clones\n",
        "        d_model = 80  # set to be 100 for SHHS dataset\n",
        "        d_ff = 120   # dimension of feed forward\n",
        "        h = 5  # number of attention heads\n",
        "        dropout = 0.1 #@param\n",
        "        num_classes = 6 #@param\n",
        "        afr_reduced_cnn_size = 30\n",
        "\n",
        "        self.mrcnn = MRCNN(afr_reduced_cnn_size) # use MRCNN_SHHS for SHHS dataset\n",
        "\n",
        "        attn = MultiHeadedAttention(h, d_model, afr_reduced_cnn_size)\n",
        "        ff = PositionwiseFeedForward(d_model, d_ff, dropout)\n",
        "        self.tce = TCE(EncoderLayer(d_model, deepcopy(attn), deepcopy(ff), afr_reduced_cnn_size, dropout), N)\n",
        "\n",
        "        self.fc = nn.Linear(d_model * afr_reduced_cnn_size, num_classes)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x_feat = self.mrcnn(x)\n",
        "        encoded_features = self.tce(x_feat)\n",
        "        encoded_features = encoded_features.contiguous().view(encoded_features.shape[0], -1)\n",
        "        final_output = self.fc(encoded_features)\n",
        "        return final_output\n",
        "    \n",
        "    # def train(self, mode=True):\n",
        "    #     \"\"\"\n",
        "    #     Override the default train() to freeze the BN parameters\n",
        "    #     \"\"\"\n",
        "        \n",
        "    #     super(AttnSleep, self).train(mode)\n",
        "    #     if self.freeze_bn:\n",
        "    #         print(\"Freezing Mean/Var of BatchNorm2D.\")\n",
        "    #         if self.freeze_bn_affine:\n",
        "    #             print(\"Freezing Weight/Bias of BatchNorm2D.\")\n",
        "    #     if self.freeze_bn:\n",
        "    #         for m in self.backbone.modules():\n",
        "    #             if isinstance(m, nn.BatchNorm2d):\n",
        "    #                 m.eval()\n",
        "    #                 if self.freeze_bn_affine:\n",
        "    #                     m.weight.requires_grad = False\n",
        "    #                     m.bias.requires_grad = False\n",
        "\n",
        "######################################################################\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I5L3BIYIZCz-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9cdbb882-60ee-4d3c-ea19-6be719690317"
      },
      "source": [
        "# Write the Skorch wrapper classifier\n",
        "# Or maybe use braindecode directly over this? But I don't see a benefit to be honest\n",
        "# Go for skorch\n",
        "\n",
        "model = AttnSleep()\n",
        "if cuda:\n",
        "    model.cuda()\n",
        "print(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "AttnSleep(\n",
            "  (mrcnn): MRCNN(\n",
            "    (GELU): GELU()\n",
            "    (features1): Sequential(\n",
            "      (0): Conv1d(1, 64, kernel_size=(50,), stride=(6,), padding=(24,), bias=False)\n",
            "      (1): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (2): GELU()\n",
            "      (3): MaxPool1d(kernel_size=8, stride=2, padding=4, dilation=1, ceil_mode=False)\n",
            "      (4): Dropout(p=0.5, inplace=False)\n",
            "      (5): Conv1d(64, 128, kernel_size=(8,), stride=(1,), padding=(4,), bias=False)\n",
            "      (6): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (7): GELU()\n",
            "      (8): Conv1d(128, 128, kernel_size=(8,), stride=(1,), padding=(4,), bias=False)\n",
            "      (9): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (10): GELU()\n",
            "      (11): MaxPool1d(kernel_size=4, stride=4, padding=2, dilation=1, ceil_mode=False)\n",
            "    )\n",
            "    (features2): Sequential(\n",
            "      (0): Conv1d(1, 64, kernel_size=(400,), stride=(50,), padding=(200,), bias=False)\n",
            "      (1): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (2): GELU()\n",
            "      (3): MaxPool1d(kernel_size=4, stride=2, padding=2, dilation=1, ceil_mode=False)\n",
            "      (4): Dropout(p=0.5, inplace=False)\n",
            "      (5): Conv1d(64, 128, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)\n",
            "      (6): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (7): GELU()\n",
            "      (8): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)\n",
            "      (9): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (10): GELU()\n",
            "      (11): MaxPool1d(kernel_size=2, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
            "    )\n",
            "    (dropout): Dropout(p=0.5, inplace=False)\n",
            "    (AFR): Sequential(\n",
            "      (0): SEBasicBlock(\n",
            "        (conv1): Conv1d(128, 30, kernel_size=(1,), stride=(1,))\n",
            "        (bn1): BatchNorm1d(30, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv1d(30, 30, kernel_size=(1,), stride=(1,))\n",
            "        (bn2): BatchNorm1d(30, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (se): SELayer(\n",
            "          (avg_pool): AdaptiveAvgPool1d(output_size=1)\n",
            "          (fc): Sequential(\n",
            "            (0): Linear(in_features=30, out_features=1, bias=False)\n",
            "            (1): ReLU(inplace=True)\n",
            "            (2): Linear(in_features=1, out_features=30, bias=False)\n",
            "            (3): Sigmoid()\n",
            "          )\n",
            "        )\n",
            "        (downsample): Sequential(\n",
            "          (0): Conv1d(128, 30, kernel_size=(1,), stride=(1,), bias=False)\n",
            "          (1): BatchNorm1d(30, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (tce): TCE(\n",
            "    (layers): ModuleList(\n",
            "      (0): EncoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (convs): ModuleList(\n",
            "            (0): CausalConv1d(30, 30, kernel_size=(7,), stride=(1,), padding=(6,))\n",
            "            (1): CausalConv1d(30, 30, kernel_size=(7,), stride=(1,), padding=(6,))\n",
            "            (2): CausalConv1d(30, 30, kernel_size=(7,), stride=(1,), padding=(6,))\n",
            "          )\n",
            "          (linear): Linear(in_features=80, out_features=80, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=80, out_features=120, bias=True)\n",
            "          (w_2): Linear(in_features=120, out_features=80, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer_output): ModuleList(\n",
            "          (0): SublayerOutput(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerOutput(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (conv): CausalConv1d(30, 30, kernel_size=(7,), stride=(1,), padding=(6,))\n",
            "      )\n",
            "      (1): EncoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (convs): ModuleList(\n",
            "            (0): CausalConv1d(30, 30, kernel_size=(7,), stride=(1,), padding=(6,))\n",
            "            (1): CausalConv1d(30, 30, kernel_size=(7,), stride=(1,), padding=(6,))\n",
            "            (2): CausalConv1d(30, 30, kernel_size=(7,), stride=(1,), padding=(6,))\n",
            "          )\n",
            "          (linear): Linear(in_features=80, out_features=80, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=80, out_features=120, bias=True)\n",
            "          (w_2): Linear(in_features=120, out_features=80, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer_output): ModuleList(\n",
            "          (0): SublayerOutput(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerOutput(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (conv): CausalConv1d(30, 30, kernel_size=(7,), stride=(1,), padding=(6,))\n",
            "      )\n",
            "    )\n",
            "    (norm): LayerNorm()\n",
            "  )\n",
            "  (fc): Linear(in_features=2400, out_features=6, bias=True)\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_bqpARgCPdQg"
      },
      "source": [
        "# This model takes in data of single channel - Paper took Pz-Oz\n",
        "# Need to create new Pytorch Dataset with single channel\n",
        "\n",
        "# Channel 0\n",
        "trainA0 = Dataset(trainB.X[:,0,:].unsqueeze(1),trainB.y.long())\n",
        "valA0 = Dataset(valB.X[:,0,:].unsqueeze(1),valB.y.long())\n",
        "\n",
        "# Channel 1\n",
        "# trainA1 = Dataset(trainB.X[:,1,:].unsqueeze(1),trainB.y.long())\n",
        "# valA1 = Dataset(valB.X[:,1,:].unsqueeze(1),valB.y.long())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F_yDnlNiP8Pq",
        "cellView": "form"
      },
      "source": [
        "#@title Weighted Cross Entropy Loss (with hard coded weights - change for continued training)\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "class weighted_CrossEntropyLoss():\n",
        "  def __call__(self,output, target):\n",
        "    # hard-coded because not sure of how to pass weights when giving loss function to skorch\n",
        "    classes_weights = [0.17, 0.22, 0.17, 0.18, 0.33, 0.17]\n",
        "    # print(output, target)\n",
        "    # print(type(output), type(target))\n",
        "    # print(type(output.long()),type(target.long()))\n",
        "    cr = nn.CrossEntropyLoss(weight=torch.tensor(classes_weights).to(device))\n",
        "    return cr(output, target)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DMe6rM0uuVjF"
      },
      "source": [
        "# Training of the model on Group 1 data using Channel 0"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iOW1XSpi0Lsn",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "outputId": "559b9a9d-755a-4898-bd2c-99a0d3fb0a47"
      },
      "source": [
        "# Retrain and save other things in checkpointing like params, etc. other than pickle.. Looks like pickle won't work for this model\n",
        "\n",
        "# starting wandb init\n",
        "# comment out for non-wandb run\n",
        "wandb_run = wandb.init(name = \"AttnSleep\", project='cnn-fulltrain', entity='sleep_hacking')\n",
        "\n",
        "# batch_size\n",
        "lr = 1e-3 #change\n",
        "batch_size = 1024\n",
        "n_epochs = 100\n",
        "\n",
        "channel = 0\n",
        "save_path = \"/content/drive/MyDrive/mne_data/AttnSleep2r_checkpoints\"\n",
        "wandb_run.config.update({\"lr\":lr,\"batch_size\":batch_size,\"channel\":channel,\"save_path\":save_path})\n",
        "\n",
        "train_bacc = EpochScoring(\n",
        "    scoring='balanced_accuracy', on_train=True, name='train_bacc',\n",
        "    lower_is_better=False)\n",
        "valid_bacc = EpochScoring(\n",
        "    scoring='balanced_accuracy', on_train=False, name='valid_bacc',\n",
        "    lower_is_better=False)\n",
        "\n",
        "cp = Checkpoint(monitor = 'valid_bacc_best',\n",
        "                f_params = \"params_{last_epoch[epoch]}.pt\", \n",
        "                f_optimizer = \"optimizer_{last_epoch[epoch]}.pt\", \n",
        "                f_criterion = \"criterion_{last_epoch[epoch]}.pt\",\n",
        "                dirname = save_path)\n",
        "train_end_cp = TrainEndCheckpoint(\n",
        "                f_params = \"params_{last_epoch[epoch]}.pt\", \n",
        "                f_optimizer = \"optimizer_{last_epoch[epoch]}.pt\", \n",
        "                f_criterion = \"criterion_{last_epoch[epoch]}.pt\",\n",
        "                dirname=save_path)\n",
        "\n",
        "# f1, accuracy\n",
        "callbacks = [('train_bal_acc', train_bacc),\n",
        "             ('valid_bal_acc', valid_bacc),\n",
        "             (\"checkpoint\",cp),\n",
        "             (\"train_end_cp\",train_end_cp),\n",
        "             (\"wandb\",WandbLogger(wandb_run)) # comment out for non-wandb run\n",
        "            ]\n",
        "\n",
        "\n",
        "# loss = getattr(weighted_CrossEntropyLoss, 'forward')\n",
        "\n",
        "clf2 = NNClassifier(\n",
        "  module = AttnSleep,\n",
        "  criterion = weighted_CrossEntropyLoss,\n",
        "  optimizer=torch.optim.Adam,\n",
        "  train_split=predefined_split(valA0),\n",
        "  iterator_train = DataLoader,\n",
        "  optimizer__lr=lr,\n",
        "  batch_size=batch_size,\n",
        "  callbacks=callbacks,\n",
        "  device=device\n",
        ")\n",
        "# callback metric f1, accuracy\n",
        "# \n",
        "print(clf)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdiv12345\u001b[0m (use `wandb login --relogin` to force relogin)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.12.2<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">AttnSleep</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/sleep_hacking/cnn-fulltrain\" target=\"_blank\">https://wandb.ai/sleep_hacking/cnn-fulltrain</a><br/>\n",
              "                Run page: <a href=\"https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/2up3ndit\" target=\"_blank\">https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/2up3ndit</a><br/>\n",
              "                Run data is saved locally in <code>/content/wandb/run-20210926_153036-2up3ndit</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized](\n",
            "  module=<class '__main__.AttnSleep'>,\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TsjbZw5gX89j",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1c67fd5d-5539-4d3a-829b-fe404c3737cb"
      },
      "source": [
        "start = time.time()\n",
        "# Model training for a specified number of epochs. `y` is None as it is already\n",
        "# supplied in the dataset.\n",
        "# clf.fit(dl.dataset.X.cpu(), y=dl.dataset.y.cpu(), epochs=n_epochs)\n",
        "clf.fit(trainA0, y=None, epochs=n_epochs)\n",
        "end = time.time()\n",
        "print(f'time taken : {end-start}')\n",
        "# saving\n",
        "# with open('/content/drive/MyDrive/mne_data/AttnSleep2_100.pkl', 'wb') as f:\n",
        "#     pickle.dump(model, f)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py:652: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)\n",
            "  return torch.max_pool1d(input, kernel_size, stride, padding, dilation, ceil_mode)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  epoch    train_bacc    train_loss    valid_acc    valid_bacc    valid_loss    cp      dur\n",
            "-------  ------------  ------------  -----------  ------------  ------------  ----  -------\n",
            "      1        \u001b[36m0.3699\u001b[0m        \u001b[32m1.3060\u001b[0m       \u001b[35m0.6611\u001b[0m        \u001b[31m0.4981\u001b[0m        \u001b[94m0.9942\u001b[0m     +  20.8272\n",
            "      2        \u001b[36m0.4655\u001b[0m        \u001b[32m0.9941\u001b[0m       0.6396        \u001b[31m0.5161\u001b[0m        1.0272     +  17.1405\n",
            "      3        \u001b[36m0.4993\u001b[0m        \u001b[32m0.9176\u001b[0m       0.6316        \u001b[31m0.5330\u001b[0m        1.0662     +  17.3853\n",
            "      4        \u001b[36m0.5298\u001b[0m        \u001b[32m0.8638\u001b[0m       0.6094        0.5319        1.0911        17.4876\n",
            "      5        \u001b[36m0.5399\u001b[0m        \u001b[32m0.8224\u001b[0m       0.6024        0.5282        1.1955        17.3755\n",
            "      6        \u001b[36m0.5625\u001b[0m        \u001b[32m0.7857\u001b[0m       0.6151        \u001b[31m0.5462\u001b[0m        1.1233     +  17.2831\n",
            "      7        \u001b[36m0.5748\u001b[0m        \u001b[32m0.7595\u001b[0m       0.6398        \u001b[31m0.5621\u001b[0m        1.0181     +  17.2074\n",
            "      8        \u001b[36m0.5817\u001b[0m        \u001b[32m0.7489\u001b[0m       \u001b[35m0.6684\u001b[0m        0.5535        \u001b[94m0.9651\u001b[0m        17.3004\n",
            "      9        \u001b[36m0.5827\u001b[0m        \u001b[32m0.7438\u001b[0m       0.6620        0.5441        \u001b[94m0.9597\u001b[0m        17.4369\n",
            "     10        \u001b[36m0.5967\u001b[0m        \u001b[32m0.7152\u001b[0m       0.6516        0.5610        1.0215        17.3592\n",
            "     11        \u001b[36m0.6005\u001b[0m        \u001b[32m0.7054\u001b[0m       \u001b[35m0.6750\u001b[0m        \u001b[31m0.5796\u001b[0m        \u001b[94m0.9165\u001b[0m     +  17.2558\n",
            "     12        \u001b[36m0.6067\u001b[0m        \u001b[32m0.6988\u001b[0m       0.6657        0.5677        0.9681        17.3573\n",
            "     13        \u001b[36m0.6105\u001b[0m        \u001b[32m0.6932\u001b[0m       \u001b[35m0.6827\u001b[0m        \u001b[31m0.5850\u001b[0m        \u001b[94m0.8705\u001b[0m     +  17.2783\n",
            "     14        0.6105        \u001b[32m0.6834\u001b[0m       0.6728        0.5788        0.9339        17.3785\n",
            "     15        \u001b[36m0.6180\u001b[0m        \u001b[32m0.6679\u001b[0m       \u001b[35m0.7020\u001b[0m        \u001b[31m0.5942\u001b[0m        \u001b[94m0.8010\u001b[0m     +  17.3060\n",
            "     16        \u001b[36m0.6235\u001b[0m        \u001b[32m0.6579\u001b[0m       0.6883        0.5864        0.8614        17.3700\n",
            "     17        \u001b[36m0.6237\u001b[0m        \u001b[32m0.6577\u001b[0m       \u001b[35m0.7071\u001b[0m        \u001b[31m0.6045\u001b[0m        \u001b[94m0.7974\u001b[0m     +  17.3084\n",
            "     18        \u001b[36m0.6280\u001b[0m        \u001b[32m0.6492\u001b[0m       \u001b[35m0.7074\u001b[0m        0.6031        0.8102        17.3996\n",
            "     19        \u001b[36m0.6342\u001b[0m        \u001b[32m0.6396\u001b[0m       \u001b[35m0.7105\u001b[0m        \u001b[31m0.6104\u001b[0m        \u001b[94m0.7917\u001b[0m     +  17.2822\n",
            "     20        \u001b[36m0.6395\u001b[0m        \u001b[32m0.6317\u001b[0m       0.7074        \u001b[31m0.6127\u001b[0m        0.8038     +  17.2750\n",
            "     21        0.6390        \u001b[32m0.6294\u001b[0m       \u001b[35m0.7243\u001b[0m        \u001b[31m0.6196\u001b[0m        \u001b[94m0.7475\u001b[0m     +  17.3830\n",
            "     22        \u001b[36m0.6456\u001b[0m        \u001b[32m0.6202\u001b[0m       0.7144        \u001b[31m0.6201\u001b[0m        0.7662     +  17.3736\n",
            "     23        \u001b[36m0.6461\u001b[0m        0.6229       0.7109        \u001b[31m0.6237\u001b[0m        0.7676     +  17.3568\n",
            "     24        \u001b[36m0.6484\u001b[0m        \u001b[32m0.6144\u001b[0m       0.7056        0.6186        0.7818        17.3837\n",
            "     25        0.6484        \u001b[32m0.6090\u001b[0m       0.7153        0.6207        0.7581        17.1799\n",
            "     26        \u001b[36m0.6555\u001b[0m        \u001b[32m0.5999\u001b[0m       0.7136        0.6206        0.7639        17.2900\n",
            "     27        \u001b[36m0.6602\u001b[0m        \u001b[32m0.5940\u001b[0m       0.7090        \u001b[31m0.6289\u001b[0m        0.7645     +  17.2721\n",
            "     28        \u001b[36m0.6605\u001b[0m        \u001b[32m0.5892\u001b[0m       0.7114        0.6226        0.7671        17.2880\n",
            "     29        \u001b[36m0.6627\u001b[0m        \u001b[32m0.5858\u001b[0m       0.7111        \u001b[31m0.6297\u001b[0m        0.7702     +  17.2112\n",
            "     30        \u001b[36m0.6636\u001b[0m        \u001b[32m0.5846\u001b[0m       0.7112        0.6293        0.7730        17.3702\n",
            "     31        \u001b[36m0.6709\u001b[0m        \u001b[32m0.5761\u001b[0m       0.7140        \u001b[31m0.6315\u001b[0m        0.7813     +  17.4084\n",
            "     32        0.6698        \u001b[32m0.5756\u001b[0m       0.7160        0.6281        0.7702        17.3965\n",
            "     33        0.6703        \u001b[32m0.5738\u001b[0m       0.7019        0.6210        0.8095        17.3646\n",
            "     34        0.6689        0.5750       0.6862        0.6214        0.8091        17.3128\n",
            "     35        \u001b[36m0.6741\u001b[0m        \u001b[32m0.5667\u001b[0m       0.7019        0.6303        0.7796        17.3601\n",
            "     36        \u001b[36m0.6795\u001b[0m        \u001b[32m0.5600\u001b[0m       0.7107        \u001b[31m0.6396\u001b[0m        0.7603     +  17.4184\n",
            "     37        \u001b[36m0.6806\u001b[0m        0.5613       0.6923        0.6287        0.7961        17.2684\n",
            "     38        \u001b[36m0.6831\u001b[0m        \u001b[32m0.5563\u001b[0m       0.6978        0.6269        0.8094        17.3972\n",
            "     39        0.6802        \u001b[32m0.5552\u001b[0m       0.6912        0.6330        0.8276        17.3614\n",
            "     40        \u001b[36m0.6865\u001b[0m        \u001b[32m0.5512\u001b[0m       0.7095        0.6332        0.7819        17.3224\n",
            "     41        0.6838        \u001b[32m0.5507\u001b[0m       0.6969        0.6323        0.8089        17.3771\n",
            "     42        \u001b[36m0.6920\u001b[0m        \u001b[32m0.5417\u001b[0m       0.6959        0.6341        0.8038        17.2918\n",
            "     43        0.6885        0.5454       0.7042        0.6364        0.7907        17.1966\n",
            "     44        \u001b[36m0.6921\u001b[0m        \u001b[32m0.5354\u001b[0m       0.6838        0.6245        0.8231        17.3757\n",
            "     45        \u001b[36m0.6944\u001b[0m        0.5357       0.7024        0.6375        0.7972        17.3135\n",
            "     46        \u001b[36m0.6980\u001b[0m        \u001b[32m0.5274\u001b[0m       0.6956        0.6319        0.7916        17.2794\n",
            "     47        \u001b[36m0.6987\u001b[0m        \u001b[32m0.5258\u001b[0m       0.6905        0.6323        0.8074        17.3043\n",
            "     48        \u001b[36m0.6993\u001b[0m        \u001b[32m0.5234\u001b[0m       0.6903        0.6321        0.8185        17.2947\n",
            "     49        \u001b[36m0.7033\u001b[0m        \u001b[32m0.5205\u001b[0m       0.6954        0.6279        0.7999        17.4192\n",
            "     50        \u001b[36m0.7039\u001b[0m        \u001b[32m0.5189\u001b[0m       0.6984        0.6350        0.8173        17.3741\n",
            "     51        \u001b[36m0.7055\u001b[0m        \u001b[32m0.5185\u001b[0m       0.7054        0.6384        0.7918        17.3900\n",
            "     52        \u001b[36m0.7063\u001b[0m        \u001b[32m0.5128\u001b[0m       0.6807        0.6270        0.8393        17.3678\n",
            "     53        \u001b[36m0.7104\u001b[0m        \u001b[32m0.5118\u001b[0m       0.7033        0.6380        0.7959        17.3180\n",
            "     54        \u001b[36m0.7110\u001b[0m        \u001b[32m0.5087\u001b[0m       0.7104        0.6338        0.7831        17.2265\n",
            "     55        \u001b[36m0.7138\u001b[0m        \u001b[32m0.5037\u001b[0m       0.7006        0.6321        0.7926        17.2808\n",
            "     56        \u001b[36m0.7150\u001b[0m        \u001b[32m0.4987\u001b[0m       0.7011        0.6250        0.8423        17.3040\n",
            "     57        0.7144        0.5010       0.6951        0.6239        0.8145        17.3159\n",
            "     58        \u001b[36m0.7154\u001b[0m        0.5000       0.7037        0.6318        0.8018        17.3960\n",
            "     59        \u001b[36m0.7172\u001b[0m        0.4987       0.7110        0.6317        0.7772        17.3318\n",
            "     60        \u001b[36m0.7248\u001b[0m        \u001b[32m0.4893\u001b[0m       0.7096        0.6311        0.7914        17.4094\n",
            "     61        0.7247        \u001b[32m0.4891\u001b[0m       0.7039        0.6287        0.8180        17.3986\n",
            "     62        \u001b[36m0.7290\u001b[0m        \u001b[32m0.4811\u001b[0m       0.7026        0.6304        0.8201        17.3710\n",
            "     63        0.7252        0.4826       0.7038        0.6288        0.8374        17.3036\n",
            "     64        \u001b[36m0.7295\u001b[0m        0.4843       0.7019        0.6335        0.8348        17.3741\n",
            "     65        0.7257        0.4861       0.7038        0.6348        0.8413        17.2781\n",
            "     66        \u001b[36m0.7304\u001b[0m        0.4814       0.7022        0.6279        0.8307        17.2808\n",
            "     67        \u001b[36m0.7366\u001b[0m        \u001b[32m0.4709\u001b[0m       0.7076        0.6381        0.8340        17.3631\n",
            "     68        0.7366        \u001b[32m0.4667\u001b[0m       0.7182        \u001b[31m0.6424\u001b[0m        0.8104     +  17.2933\n",
            "     69        \u001b[36m0.7451\u001b[0m        \u001b[32m0.4592\u001b[0m       0.7129        0.6398        0.8145        17.3918\n",
            "     70        0.7414        0.4606       0.7180        \u001b[31m0.6450\u001b[0m        0.8092     +  17.2746\n",
            "     71        0.7443        0.4673       0.7164        0.6413        0.8057        17.3994\n",
            "     72        \u001b[36m0.7483\u001b[0m        \u001b[32m0.4569\u001b[0m       0.7059        0.6350        0.8402        17.2628\n",
            "     73        0.7476        \u001b[32m0.4542\u001b[0m       0.7163        0.6421        0.8268        17.2656\n",
            "     74        \u001b[36m0.7499\u001b[0m        \u001b[32m0.4494\u001b[0m       0.7112        0.6358        0.8193        17.3129\n",
            "     75        \u001b[36m0.7537\u001b[0m        \u001b[32m0.4469\u001b[0m       0.7110        0.6360        0.8261        17.2648\n",
            "     76        0.7537        0.4485       0.7123        0.6295        0.8315        17.4278\n",
            "     77        \u001b[36m0.7567\u001b[0m        \u001b[32m0.4448\u001b[0m       0.7166        0.6370        0.8478        17.3120\n",
            "     78        0.7561        \u001b[32m0.4440\u001b[0m       0.7068        0.6279        0.8632        17.3565\n",
            "     79        \u001b[36m0.7628\u001b[0m        \u001b[32m0.4345\u001b[0m       0.7091        0.6331        0.8813        17.1590\n",
            "     80        0.7601        0.4361       0.7073        0.6338        0.8742        17.3233\n",
            "     81        0.7602        \u001b[32m0.4329\u001b[0m       0.7046        0.6305        0.8612        17.3660\n",
            "     82        \u001b[36m0.7658\u001b[0m        \u001b[32m0.4295\u001b[0m       0.7062        0.6331        0.8604        17.4122\n",
            "     83        \u001b[36m0.7701\u001b[0m        \u001b[32m0.4251\u001b[0m       0.7096        0.6417        0.8764        17.4058\n",
            "     84        0.7686        0.4268       0.7029        0.6285        0.8834        17.3126\n",
            "     85        \u001b[36m0.7714\u001b[0m        \u001b[32m0.4238\u001b[0m       0.7066        0.6312        0.8562        17.3994\n",
            "     86        0.7705        \u001b[32m0.4187\u001b[0m       0.6941        0.6246        0.9172        17.3705\n",
            "     87        0.7707        0.4229       0.7077        0.6335        0.9112        17.2910\n",
            "     88        \u001b[36m0.7732\u001b[0m        \u001b[32m0.4172\u001b[0m       0.7023        0.6365        0.8965        17.2851\n",
            "     89        \u001b[36m0.7801\u001b[0m        \u001b[32m0.4091\u001b[0m       0.7074        0.6386        0.8787        17.3824\n",
            "     90        \u001b[36m0.7839\u001b[0m        \u001b[32m0.4027\u001b[0m       0.6949        0.6284        0.9164        17.2568\n",
            "     91        0.7800        0.4131       0.6965        0.6294        0.9172        17.2799\n",
            "     92        0.7823        0.4070       0.6973        0.6352        0.9184        17.3788\n",
            "     93        0.7832        0.4066       0.6997        0.6337        0.9199        17.2973\n",
            "     94        \u001b[36m0.7885\u001b[0m        \u001b[32m0.3989\u001b[0m       0.7018        0.6316        0.9022        17.3914\n",
            "     95        \u001b[36m0.7891\u001b[0m        0.4000       0.7038        0.6340        0.8873        17.2783\n",
            "     96        \u001b[36m0.7900\u001b[0m        \u001b[32m0.3922\u001b[0m       0.7038        0.6283        0.9040        17.3059\n",
            "     97        0.7900        0.3929       0.7081        0.6330        0.9146        17.2725\n",
            "     98        \u001b[36m0.7963\u001b[0m        \u001b[32m0.3864\u001b[0m       0.7069        0.6344        0.9141        17.1947\n",
            "     99        \u001b[36m0.7968\u001b[0m        \u001b[32m0.3848\u001b[0m       0.7009        0.6329        0.9184        17.4075\n",
            "    100        \u001b[36m0.7999\u001b[0m        \u001b[32m0.3816\u001b[0m       0.7022        0.6298        0.9432        17.2332\n",
            "time taken : 1749.6795363426208\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BiLb1HKnYReb"
      },
      "source": [
        "# Load from checkpoint\n",
        "\n",
        "# clf.load_params(checkpoint=cp) # soesn't seem to have loaded it"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x6LgFzkoYaP7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 358
        },
        "outputId": "919a62a0-745c-4a92-cb79-c9427c23948b"
      },
      "source": [
        "training_viz(clf)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABZgAAAHwCAYAAAArRQrgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdf3zVdf3//9vZL7adwYANBgKKAvMcNFNETTRnmfYm+mEm9s5+WJmiZuU7tS/2451p76If9rZPiZFllpkZZr4z0izTlWIpaj+U1xEBUX7JjyGDbWw7O+d8/9iGA8d2Nradbed2vVx2Yee8nq/neWzM1e578HiGUqkUkiRJkiRJkiT1VE6mC5AkSZIkSZIkDU0GzJIkSZIkSZKkXjFgliRJkiRJkiT1igGzJEmSJEmSJKlXDJglSZIkSZIkSb2Sl+kCBlpOTk6qqKgo02VIkiRJkiRJyhINDQ2pVCo1LJt9sy5gLioqor6+PtNlSJIkSZIkScoSoVBoT6Zr6C/DMjWXJEmSJEmSJPU/A2ZJkiRJkiRJUq8YMEuSJEmSJEmSesWAWZIkSZIkSZLUK1l3yJ8kSZIkSZKkVqlUing8TjKZzHQpQ1JOTg75+fmEQqFMl5IxdjBLkiRJkiRJWailpYWamhri8XimSxmy4vE4NTU1tLS0ZLqUjLGDWZIkSZIkScoyqVSKnTt3UlZWltXdt32huLiYmpqarP1c2sEsSZIkSZIkZZl4PE5RUVFWBqJ9LRQKUVhYmLWd4AbMkiRJkiRJUpZJJpPk5uZmuoxhIzc3N2vnWBswS5IkSZIkSdJByOZOcANmSZIkSZIkSVKvGDBLkiRJkiRJknrFgFmSJEmSJElSVvvoRz/KO9/5zkyXMSSFUqlUpmsYUOFwOFVfX5/pMgZMYzxBYb4D2yVJkiRJkvSaxsZGAAoLCzNcSc90N+v4ggsu4LbbbuvxvrW1taRSKUaPHt2rurr7fIZCoYZUKhXu1eaDXF6mC1Dfq22Is7h6NXev2EBNfTNl4QLOnT2Zy6qmU1qcn+nyJEmSJEmSpF7ZvHnz3vd/97vfcdFFF+3zXFFR0T7r4/E4+fnd52GlpaV9V2SWcUTGMFPbEGf+kuUsqV5LTX0zADX1zSypXsv8JcupbYhnuEJJkiRJkiSpdyZMmLD3rb3buP1xY2Mjo0eP5s477+Stb30rRUVFLFmyhJqaGj7wgQ8wefJkioqKOOqoo/jJT36yz777j8g4/fTTueyyy/j85z9PeXk548eP56qrriKZTA7oxzsUGDAPM4urV7NqS12n11ZtqePm6jUDXJEkSZIkSZI0cK655houu+wyVq5cydlnn01jYyOzZs3id7/7Hc899xyf+cxnWLBgAQ899FCX+9xxxx3k5eWxfPlyvv/973PjjTdy1113DdBHMXQ4ImOYWbpiQzfX17NwbmSAqpEkSZIkSdJQ0t2M4/7Q12fEfepTn+Lcc8/d57mrr7567/sXX3wxf/7zn7nzzjs544wzDrjPzJkzue666wCorKzklltu4aGHHuIDH/hAn9Y71BkwDyON8QQ72sZiHEhNfbMH/0mSJEmSJKlTfR32ZsLs2bP3eZxIJFi0aBF33XUXGzdupKmpiebmZk4//fQu9znmmGP2eXzIIYewdevWvi53yHNExjBSmJ/L2HBBl2vKwgWGy5IkSZIkSRq2wuHwPo+//e1vc8MNN3D11Vfz0EMP8Y9//IOzzz6b5uauGzX3PxwwFAo5g7kTBszDzPzZk7u5PmWAKpEkSZIkSZIy79FHH+Vd73oXH/7whzn22GOZNm0aq1atynRZw4YB8zBzWdV0KitKOr1WWVHCpVXTBrgiSZIkSZIkKXMqKyt56KGHePTRR4nFYlx++eW8+OKLmS5r2DBgHmZKi/NZumAOl1RNI6dtJntxQS6XVE1j6YI5lBbnd72BJEmSJEmSNIx88Ytf5MQTT2Tu3LmcdtpphMNhPvjBD2a6rGEjNBwGd/dEOBxO1dfXZ7qMAXHeksd54sUdXFI1jYVzI5kuR5IkSZIkSYNEY2MjAIWFhRmuZHjo7vMZCoUaUqlUuNOLQ5wdzMPYuJIRAGyva8pwJZIkSZIkSZKGIwPmYay8pAAwYJYkSZIkSZLUPwyYh7FyO5glSZIkSZIk9aO8TBeg/lM+sjVg3rbbgFmSJEmSJEkaaoJINBe4FvgQMBHYDNwBXBuNBS1ta0LAl4GLgTHA34FPRmPBcwNRox3Mw1h7B3NNXTPJZHYd5ihJkiRJkiQNA/8f8Eng00AE+Ezb42s6rPkccCXwKeAEYCvwxyASHTkQBdrBPIy1z2BuSaao3RNnTLggwxVJkiRJkiRJ6oE5wH3RWHBf2+N1QST6W+Ak2Nu9fAWwKBoLft323AW0hsznA0v6u0A7mIex9g5mcA6zJEmSJEmSlEGhUChU2eGtLM37HgXeEkSiEYAgEp0JvBX4fdv1w4EJwIPtN0RjwR7gL7SG0/3OgHkYGzfytYB5mwGzJEmSJEmSlCl5wPMd3j6V5n3fAG4HVgaRaBx4DvhpNBYsbrs+oe3PLfvdt6XDtX7liIxhrDA/l5IRedQ1tbC9rjnT5UiSJEmSJEnZqgU4usPjmjTvez/wEVrHXTwHHAt8N4hEX4zGgh/3bYm9YwfzMNc+h3n7bjuYJUmSJEmSpGuvvZajjz76gI87c/nll3P66acfzMumUqnUqg5v6QbM3wK+HY0Fv4zGgn9HY8HtwHd47ZC/V9r+rNjvvooO1/qVAfMw1z6H2RnMkiRJkiRJGure/e53c8YZZ3R6LQgCQqEQDz74YKfXD+Sqq66iurq6L8rrD8VAYr/nEryW675Ia5B8ZvvFIBItBN4MLB+IAh2RMcwZMEuSJEmSJGm4uPDCC3nve9/LunXrmDp16j7XfvzjH3PYYYfxtre9rUd7lpSUUFJS0odV9qn7gIVBJPoirSMyjgM+C/wMIBoLUkEkeiPw+SASjQGrgC8CdcAvBqJAO5iHufKRbSMynMEsSZIkSZKkIW7evHlUVFTwk5/8ZJ/n4/E4t99+Ox/72Me46KKLOPzwwykqKmLGjBl885vfJJlMHnDP/UdkJBIJrrrqKsaMGcOYMWO44oorSCT2byIeMJ8C7gYWAwFwA3AL8IUOa74J/C9wE7ACmAicFY0FuweiQDuYh7lxJYWAHcySJEmSJEka+vLy8rjgggu47bbb+PKXv0xOTmv/7H333cf27dv5+Mc/zi233MKvfvUrxo0bxxNPPMHFF19MWVkZF154YVqvccMNN3DLLbdwyy23cMwxx3DTTTdxxx13MGvWrP780DrVFhJf0fZ2oDUp4Nq2twFnB/Mwt7eD2UP+JEmSJEmS1I1rr72Wa6+9FoDKykpWrVrFU089xfHHHw/AlVdeyQ033ADAIYccwqZNm3jkkUf2HoB38cUX88Mf/hCAkSNHsnv3bu677z7e9a53AXD++efzi1+0Tm4IhUK9qvHCCy/k5Zdf5k9/+tPe53784x9z1llnMWXKFK677jpOOOEEpk6dynnnnccll1zCnXfemfb+N954I5/73Oc477zziEQifPe732XChAm9qjUb2ME8zL02g7mZVCrV6/9wJUmSJEmSNPy1h8sAq1at2vv+U089BbA3XAbYtGkT0Bo0P/LIIwB7w2WA3btbJzS8613v2hswt4fLAKlUqlc1zpgxg6qqKm699VbOOussNm3axB/+8Ad++ctfAvCDH/yAH/3oR7z00kvs2bOHeDzOYYcdltbetbW1bN68mZNPPnnvczk5OZx00kmsX7++V/UOd3YwD3PtAXNzIsmuxpYMVyNJkiRJkiQdvAsvvJB7772XHTt2cNtttzF27Fje8573cNddd3HFFVfw0Y9+lD/84Q/84x//4LLLLqO52fPJ+osB8zA3ri1gBucwS5IkSZIkaXg499xzKSws5Oc//zm33norH/nIR8jPz+fRRx/lpJNO4vLLL2fWrFlMnz6dNWvWpL1vaWkpEydO5G9/+9ve51KpFE888UR/fBjDggHzMNc+gxmcwyxJkiRJkqThoaioiPPPP59rr72WNWvW7D3Ar7Kykqeffpr777+fF154geuvv57q6uoe7f2Zz3yGb37zm9x99908//zzXHHFFWzevLk/PoxhwYB5mCsuyKO4IBdoncMsSZIkSZIkDQef+MQnePXVV5kzZw7RaBSABQsWcN5553H++edzwgknsG7dOq688soe7XvllVfysY99jE984hOcdNJJJJNJPvjBD/bHhzAshHo7THuoCofDqfr6+kyXMaBO++bDvLyjga+8+ygumDM10+VIkiRJkiQpwxobGwEoLCzMcCXDQ3efz1Ao1JBKpcIDWdNAsYM5C5SXtI7J2OaIDEmSJEmSJEl9yIA5C5S3HfTnIX+SJEmSJEmS+pIBcxYoH2nALEmSJEmSJKnvGTBngfYO5m0e8idJkiRJkiT1uWw7564jA+YsMK5tBvN2ZzBLkiRJkiQJyMnJIZFIZLqMYSORSJCTk51Ra3Z+1Fmm4wzmbP5tiiRJkiRJklrl5+ezZ88es6I+kEqlaGxsJD8/P9OlZERepgtQ/2ufwdzUkqSuqYWRhdn5xS5JkiRJkqRWoVCI0aNHU1NTQ2FhIbm5uYRCoUyXNaSkUikSiQSNjY2MHj06az9/BsxZoL2DGWB7XbMBsyRJkiRJksjLy6OsrIx4PE4ymcx0OUNOKBSioKCAcDicteEyGDBnhXEjOwbMTRxeHs5gNZIkSZIkSRos2kNSqbecwZwFwgW5FOa3/lV70J8kSZIkSZKkvmLAnAVCodA+B/1JkiRJkiRJUl8wYM4S7QHztrrmDFciSZIkSZIkabgwYM4SdjBLkiRJkiRJ6msGzFli3MjWYe3OYJYkSZIkSZLUVwyYs4QdzJIkSZIkSZL6Wt5AvtjUhctOA64CjgcOAT62btG827pYfzrwX8CJQCmwGrhx3aJ5t/Z7scPMawGzM5glSZIkSZIk9Y2B7mAuAZ4FPgPsSWP9HODfwLnA0cDNwA+nLlx2fr9VOEwN1Q7mxngi0yVIkiRJkiRJOoAB7WBet2je74HfA0xduOy2NNZ/bb+nbp66cNlbgPcBv+jzAoex8pLWGcwNzQnqm1oIjxjQv/oeqW2Is7h6NXev2EBNfTNl4QLOnT2Zy6qmU1qcn+nyJEmSJEmSJLUZvCnjgY0CNvTkhlAoVAaUARQVFfVHTYNe+cgRe9/fXtc0aAPm2oY485csZ9WWur3P1dQ3s6R6LQ/HtrJ0wRxDZkmSJEmSJGmQGFKH/E1duOydwBnAD3t466eA54HnW1pa+ryuoaB9RAYM7jEZi6tX7xMud7RqSx03V68Z4IokSZIkSZIkHciQCZinLlx2Cq1jMT69btG8J3p4+/eAI4Ej8/IGZ+dufxtVmEdBbutf97bdg/egv6Urum5OX7pi/QBVIkmSJEmSJKk7QyJgnrpw2anA/cB/r1s07+ae3p9KpWpSqdSqVCq1KhQK9X2BQ0AoFNo7h3mwdjA3xhPsqO86/K6pb/bgP0mSJEmSJGmQGPQB89SFy06jNVy+dt2ieTdmup6hrH0O82ANmAvzcxkbLuhyTVm4gML83AGqSJIkSZIkSVJXBnRexNSFy0qA6W0Pc4BDpy5cdiywY92ieS9PXbjs68CJ6xbNO6Nt/enAMmAx8IupC5dNaLs3sW7RvG0DWftw0D6HebAGzADzZ09mSfXaLq5PGcBqJEmSJEmSJHVloDuYZwPPtL0VAV9pe/+6tusTgWkd1n8UKAauAjZ3eHtyYModXvaOyBjEM5gvq5pOZUVJp9cqK0q4tGpap9ckSZIkSZIkDbxQKpXKdA0DKhwOp+rr6zNdRkZ86w8xbnp4DbMPG8Pdl87JdDkHtHJzLe/47qN7H5eFC5g/ewqXVk2jtDg/g5VJkiRJkiRJPRcKhRpSqVQ403X0hwEdkaHMGgojMgCOrBjFvZ88hcdWb+eCkw+jpDCfV2obeWTVVt5z7KRMlydJkiRJkiSpjQFzFnktYB68IzIAcnNCHDtlNMdOGQ3A39bWcP4tfyMFzJ46lkmjizJboCRJkiRJkiRg4GcwK4PaA+a6phYa44kMV5O+4w4dzaiifEaOyGPVlt2ZLkeSJEmSJElSGzuYs8i4kQV739+2u4kpY4szWE3ntu5q5NWGOJUVJYRCIQBG5OXy8wtPYvr4EgrzczNcoSRJkiRJkqR2djBnkfYOZhi8c5h//fRG3n7jXzj7psf2ef7oSaWGy5IkSZIkSdIgY8CcRUqL8snPbe0KHqxzmP+5ficAMypGZrgSSZIkSZIkSd1xREYWCYVClIVH8MquxkHbwbz4g7MIXtlFQe7rf/expznBPc9s4DdPb+SWj8xmTLigkx0kSZIkSZIkDRQ7mLNMedsc5u27B2fAnJMT4qhDSjvtYG5JJvnq7wJWvPQqv3lmYwaqkyRJkiRJktSRHcxZpn0O82DtYO7KyMJ8zpk1iV2NLbxxyuhMlyNJkiRJkiRlPQPmLNMeMG8bhAHzL594maMOKeWoQ0aRkxPqdM1Xzz6aUKjza5IkSZIkSZIGliMysszeDubdg+uQv227m1h4z7951/cf5a+rtx9wneGyJEmSJEmSNHgYMGeZ8pK2GcyDrIN5/asNTBhVSEFuDidOHZvWPZt27mFPc6KfK5MkSZIkSZJ0II7IyDLjRg7OERmzDh3D49e8lU21jRQV5Ha5NplMcfHtT/FQbAvfeN8xnDd7ygBVKUmSJEmSJKkjO5izTPuIjN2NLTTGB1f3bygUYtLoom7X5eSEyMsJkUrB7/+9eQAqkyRJkiRJktQZO5izTHvADFBT35xWoDsYLag6gnccM5GzZlZkuhRJkiRJkiQpaxkwZ5n2ERkA23c3DYqA+YFnXyH2yi7eGhnPMZNHp3XPcYeO4bhDx/RzZZIkSZIkSZK64oiMLDO6KJ/cnBAweA76u/up9dz4pxdYUr0206VIkiRJkiRJ6gED5iyTkxOiLFwADJ6A+fDyMFPGFnHK9PIe31u7J87tf3uJ5Wu290NlkiRJkiRJkrriiIwsVF4ygq27m9he15zpUgD4wryZfGHeTBLJVI/v/cwvn+GR57dx+pHjmDOt5wG1JEmSJEmSpN6zgzkLlbfNYd62e3B0MLdrH93RE+fMmkxBXg5jiwtI9iKgliRJkiRJktR7djBnofKSwTUi42C8/agKTptxBqOLCzJdiiRJkiRJkpR17GDOQuNKWjuYMx0w726M897Fj/GtP8So6WUtI/Jy9wmXG+OJvipPkiRJkiRJUjfsYM5C5XsD5szOYP7b2h088/JO/rl+JxefNq3X+9Q2xFlcvZq7V2ygpr6ZsnAB586ezGVV0yktzu/DiiVJkiRJkqSBE0Si64DDOrn0+2gsmNe25jLgamAi8BxwRTQW/HWgarSDOQuVjxwcIzImjynio3Om8q43HkJpUe+C4NqGOPOXLGdJ9Vpq6lsD85r6ZpZUr2X+kuXUNsT7smRJkiRJkiRpIJ1Aa3Dc/jYLSAG/Aggi0fcD3wW+BhwHLAfuDyLRQweqQDuYs1B7B/POhjjxRJL83Mz8niE6cRTXvvuog9pjcfVqVm2p6/Taqi113Fy9hoVzIwf1GpIkSZIkSdJBCoVCocoOj2tSqVRNdzdFY8G2jo+DSPRCYBdtATPwWeC2aCy4pe3xp4JI9D+AS4FrDr7s7hkwZ6H2gBmgpq6ZCaWFGazm4CxdsaGb6+sNmCVJkiRJkpRpecDzHR5/Bbi2JxsEkWgIuBD4eTQW7Aki0QLgeODb+y19EJhzgD1m9eQ12/w7GgsOOCbAgDkLdQyYt9c1ZSRgbkkkyTvIzunGeIId9V3Pka6pb6YxnqAwP/egXkuSJEmSJEk6CC3A0R0ed9u93IkzgcOB9m7lciAX2LLfui3A2w6wxwpaR2yE0nzNJFAJrD3QAgPmLDQ2XEBOCJIp2LY7M3OYP/urf7Jqy24+furhnDd7Sq/2KMzPZWy4oMuQuSxcYLgsSZIkSZKkTEulUqlVB7nHRcCT0Vjwz4Pc5yRgW7erWkPoZ7tbZMCchXJzQowNF7C9rpltGTjoL5lM8djq7dTUN7O7seWg9po/ezJLqg/4CxTm9zK8liRJkiRJkgaLIBIdD7wH+GSHp7cDCaBiv+UVwCsH2KoaWB2NBTvTfN2/AHu6WpOZ092Uce1jMrZnIGBOAf/7/mO5+LQjeMuR4w5qr8uqplNZUdLptcqKEi6tmnZQ+0uSJEmSJEmDwEeBJuDO9ieisaAZeIrW0RkdnQks72yTaCx4S7rhctv6d0Rjweau1tjBnKVaA+bdbN/d9Qzj/pCbE+K0ynGcVnlw4TJAaXE+SxfM4ebqNSxdsZ6atnEZ40eOYOmCOZQW5x/0a0iSJEmSJEmZ0na43yeAX0ZjQd1+l78D3B5Eok8AjwGXAIcAP+jF64SB3Ggs2NWT+wyYs1R5SQGQmQ7mvlZanM/CuREWzo3w9EuvsmHnHk6ZVma4LEmSJEmSpOHgdGAG8KH9L0RjwV1BJFoGfBGYSOvM5HdEY8FL6W4eRKJHArcDs4FUEIk+B1wQjQXPpHO/AXOWGjcyMyMymloS7KhvZmJpUb/sP+uwMcw6bEy/7C1JkiRJkiQNtGgseJjWA/cOdH0xsPggXmIJcAutQfYI4Grgp8Ax6dzsDOYslakZzE+++Conf/3PvO071TQ0H9wBf5IkSZIkSZJ6JohEf97W9dxuInBHNBY0RGPBq8BdwOR09zNgzlKvBcwDO4P5sTXbAcgNhSgu6L8G+l2NcapXbeu3/SVJkiRJkqQh6u/Ak0Ek+uG2x78Anggi0W8Ekeh3gN/S2sGcFkdkZKnythEZrzY005JIkpc7ML9ruKTqCE6bMY7GeKLfXuPZjbW856bHSCRTPLbwrUwa3T/jOCRJkiRJkqShJhoLvhdEovcCN7WFzJcAK4C30jqK47PAPenuZ8CcpdoP+UulYEd9M+NHFfbba9U2xFlcvZq7V2ygpr6ZsnAB586ezKxDx/TLQXyVFSMZkZdDYzzBsxtrDZglSZIkSZKkDqKxYD3w7iASPQ/4I61zmK+OxoJkT/dyREaWGtc2IgNgWz/OYa5tiDN/yXKWVK+lpr51HEdNfTNLqtcyf8lyahviff6aBXk53H7hiTzz32fx9qMm9Pn+kiRJkiRJ0lAXRKJl0VjwK2AWMJ3WsRmzerqPAXOWGhsuINR29mR/zmFeXL2aVVvqOr22aksdN1ev6ZfXPf6wsZQW9X13tCRJkiRJkjSUBZHoGUEkugXYFkSiG4CZ0VhwMa2jMW4PItEbgkg07ZEABsxZKi83hzHFrWMytu/uvw7mpSs2dHN9fb+9tiRJkiRJkqTXuQn4JlAMXA7cCBCNBdXAccAe4B/pbmbAnMXa5zBv76cRGY3xBDvqu+6Orqlv7rcD/16tb+Z3/9rEX1Zt65f9JUmSJEmSpCFoIrAsGgsagQeAce0XorGgORoLvgick+5mBsxZrLxtDnN/BcyF+bmMDRd0uaYsXEBhfm6/vP51v1vJ5b94hh89+mK/7C9JkiRJkiQNQb8F7g4i0a8BDwK/339BNBY8l+5mBsxZ7LWAue9nMKdSKX72+Drec+whXa6bP3tKn792u1Oml5OfGyLUVo8kSZIkSZIkLgSWAKXAz4ErDmazvL6oSENTf3Ywf/43z3LnEy9z0uFjmDG+hBe2vv6gv8qKEi6tmtbnr91u3hsmMvfoCYRH+GUuSZIkSZIkQesYDOB7fbWfHcxZrHxk6/iKbf1wyN8bJpUCMH5UEXdcdBKXVE2jrG1cRlm4gEuqprF0wRxKi/P7/LXbFRXkGi5LkiRJkiRJbYJI9MQgEk17Xm0QiR4fRKJdBnimb1msPzuYzz/pUCaNKeLN08vJyQmxcG6EhXMjNMYT/TZzWZIkSZIkSVKXHgcmANvSXP8wcCyw9kALDJiz2Li2gHlHfTOJZIrcnFCP92gPjNduq+P5V3Yz9w0T916rqhz3uvUDHS7vaU5w/7ObeXT1dv77nTMZXdz1oYOSJEmSJEnSMBYCvh5Eog1pru82TDNgzmLtHczJVGvIPG7kiLTuq22Is7h6NXev2EBNfTOlRfk0xhO0JJL89OMnceqM8v4su0dakkmuvvtfJJIpzppZwX8cPbH7myRJkiRJkqTh6S9ATw5FexzY09UCA+Ys1jFQ3l7XlFbAXNsQZ/6S5aza8tqhfbV74gDkhCCeTPZ9oQdhZGE+Jx9RRigExQV+uUuSJEmSJCl7RWPB6X29p4lbFisrea3DPd05zIurV+8TLneUTMHf1+7gLUeO75P6+srtF55IKNTz8R+SJEmSJEmSupaT6QKUOfm5OYwubj0EMt2AeemKDd1cX3/QdfU1w2VJkiRJkiSpfxgwZ7n2Oczbdzd3u7YxnmBHfdfrauqbaYwn+qQ2SZIkSZIkSYObAXOWK28bk5FOB3Nhfi5jw10fHFkWLqAwP7dPautLf31hG5f/4mm+eO+/M12KJEmSJEmSNGwYMGe59g7mbWmOyJg/e3I316ccdE39YfXWOn73r838/t+vkEymMl2OJEmSJEmSNCx4yF+W2zsio677ERkAl1VN5+HY1k4P+qusKOHSqml9Wl9fefOMcZwRGc8p08uJJ5OMyBl8XdaSJEmSJEnSQAoi0TcAC4BpwMejsWBzEImeDbwUjQXPpLOHHcxZbtzI9hnMXXcwN8YT/GP9TkqL81m6YA6XVE2jrG1cRlm4gEuqprF0wRxK2w4NHGymjy/hxx89gY+fejgj8gyXJUmSJEmSlN2CSPQs4ElgEvBWoKjt0jTgy+nuYwdzlkt3BvPXfx9w+99e4r/eVsmnzpjBwrkRFs6N0BhPDMqZy5IkSZIkSZK6dD3w2WgsWBxEors7PP8IcGW6m9jBnOXaR2TU1DcfcDZxQ3MLy9fUkExBcyK5z7WhGC63JJIknMMsSZIkSZKk7HY08PtOnt8BjE13EwPmLNceMCeSKXbuiXe6prggj99efioL50b49BkzBrK8PhVPJFlw+wqOu/6P/G1tTabLkSRJkiRJkjJpB63jMfY3C9iQ7iYGzIHWl8YAACAASURBVFmuvG0GM3Q9JqOoIJdLqqaRnzt0v2Tyc3N4cXs9uxtbeHT19kyXI0mSJEmSJGXSL4BvBZHoZCAF5AWRaBXwbeBn6W7iDOYs135QH7Qe9FdZMXLv4z+t3MIJU8cO2oP7euOiNx9BU0uSqspxmS5FkiRJkiRJyqQvArcBLwEhYGXbn78A/ifdTQyYs1xhfi4jC/PY3djCtg4dzP/asJNLfv4U40eO4CcfO5EjJ4zsYpehY/7sKZkuQZIkSZIkScq4aCyIAx8MItEv0ToWIwd4JhoLXujJPgbMYlzJiNaAeXcTjfEEhfm5vLi9npycEPl5OUwaU5TpEiVJkiRJkiT1oSAS/W/g29FYsBZY2+H5IuDqaCy4Lp19DJjF6KLWERjffvB5vrosoCxcwLmzJ3PHJ06kKD+PkhHD78ukuSXJpp17mFoeznQpkiRJkiRJUiZ8GfgB0LDf88Vt19IKmIfuiW3qE7UNcVZtrQOgMZ4EoKa+mSXVa/nCb55lypjiTJbXL+59ZiNv/MqDfOTWJzJdiiRJkiRJkpQpIVoP99vfccCOdDcZfq2p6pHF1aupa2rp9NqqLXXcXL2GhXMjA1xV/5oytog98QQv72hg4849TBo9uEaAtI8pkSRJkiRJkvpaEInupjVYTgFrg0i0Y8icCxTS2tmcFgPmYS7Z2EhOYeEBry9dsaHL+5euWD/sAuZjJo/mm+cew6nTyxkbLsh0OUBrJ/ni6tXcvWIDNfXNe8eUXFY1ndLi/EyXJ0mSJEmSpOHjclq7l28FvgDUdrjWDKyLxoLH093MgHkYStTWUnPLLey85zckduwgd+xYRp/zXsouuojc0tK96xrjCXbUN3e5V01987DrqG1oSrBmWx3fuD82KMLc2oY485csZ9WWur3PtY8peTi2laUL5hgyS5IkSZIkqU9EY8FPAYJI9EVgeTQWxA9mv1Aq1dmYjeErHA6n6uvrM11Gv0nU1vLShz5E0wurX3dtxIzpHPbzn+8TMs+6/o9dhsxl4QKe+tKZ/VJrJnQW5rarrCjJSJj79fsDllSvPeD1S6qmDbsuckmSJEmSpGwSCoUaUqlUONN1HEgQiU4A9vmn/tFY8HI693rI3zBTc8stnYbLAE0vrKbmRz/a57lzj5/c5X7zZ0/ps9oGg8XVqzsNl+G1mdMDLZ0xJZIkSZIkSVJfCiLRUUEk+tMgEt0DbARe3O8tLQbMw8zOX9+T9vV/rt/JH1duOeAhd5UVJVxaNa1P68u0wRbm9mRMiSRJkiRJktSHbgDeCJwNNALnA1cDG4D3p7uJAfMwkmxsJPHqq12uSezYQbKpiXgiyeV3Ps2L2+spzN/3y6AsXMAlVdOG3ezfwRjmFubndnvQYFm4YFjNwJYkSZIkSdKgMBf4VDQW/AFIAE9FY8F3gIXAgnQ3MWAeRnIKC8kdM6bLNbljx5IzYgT5uTn8v/88jqllxdz0wVmUjGg97/E7572Rp750JgvnRoZVuAyDN8ydPzu7xpRIkiRJkiRpUBgNvNT2fi1Q1vb+48CcdDcxYB5mRr/vnLSvH3foGP702SoiE0ZRXtIavNbuOahDIwe9wRjmXlY1ncqKkk6vDccxJZIkSZIkSRoU1gBHtL0fAP8ZRKIh4BxgR7qbGDAPM2UXXcSIGdM7vTZixnTWnHEOqVRq73N5ua1fAuUlIwDYXtfU/0VmUFdh7rRx4YyEuaXF+SxdMIdLqqYxuui1rvH3Hjdp2I0pkSRJkiRJ0qBxG3BM2/uLaB2L0Qx8C/hGupuEOoaN/W3qwmWnAVcBxwOHAB9bt2jebd3c8wbg+8CJtCbnS4Dr1y2a16vCw+Fwqr6+vje3DhmJ2lpqfvQjXr3zlyTr6gAY+/GP8cfj5vLFh17mAyceynXvOYr83Nd+v3DJ7U/xwHOv8P7ZU/jGucccaOthobYhzs3Va1i6Yj019c2MKc6nLDyCV2r3UP25t1DWFrYPhG27m/jkHU9zwZypvP2oCnJzQiy6P2DWYWM5dXo54bbRJZIkSZIkSRq6QqFQQyqVCme6jq4EkeihwGzghWgs+He69w10elUCPAv8rO2tS1MXLhsF/BH4C3ACEAF+AtTTesqhOpFbWsr4K69k9Pv/kzVvexsAo94xj+X/bATgldo9hPa7p3xk64iM4d7BDK0dwwvnRlg4N0JjPEF9UwtV33qEuuYEd/z9ZT59xowBq+Wny9fxxLodBK/sourIMygZkcc175g5YK8vSZIkSZKk7BNEovnAo8BHorHgeYBoLHgZeLmnew1owLxu0bzfA78HmLpw2W1p3PJBoBi4YN2ieXuAZ6cuXBYBPjt14bLv9LaLOVvkTzqEnFGjSO7aRWOwku9/4FxuPfRF/vPEQ/eOxmjXPiJj6+7GTJSaMYX5uRTm5/K5/ziSgtwczj2+6xnNfe3UGeX8a2MtMyeO2nvQoiRJkiRJktSforEgHkSihwMHna8O9kTrZOCvbeFyuz8A1wNTgRfT2SQUCpXRdgpiUVFRH5c4+NQ2xFlcvZq7V2zgqoLxHMsulj+wnFPnnc0n3nxEp+ufeulVAP69cRfHX/9Hzp09mcuqpmfN/N+PnDw1I6/7piPKeNMRZSSTr/9vuaauiZZkiopRhRmoTJIkSZIkScPcT4GLgKsPZpPBHjBPADbs99yWDtfSCpiBTwFfBmhpaembygap2oY485csZ9WW1tnLq0dP4tjtq2laGTB/yfLXHRq3/3qAmvpmllSv5eHYVg+ZGyA5OfsOLbnil89w7z82ceGph/OldzoyQ5IkSZIkSX0uDHwwiETPBJ6idSzxXtFY8Ol0Nsnpfsmw8D3gSODIvLzBnqkfnMXVq/cJi1ePbh35cPiuTax+ZRc3V6/pcn1Hq7bUvW79cNfckuSny9fxhd+kPce8V1Zu2sWWXQceR1JR2tq1/Le1Nf1ahyRJkiRJkrJWFHgaeBU4AnhDh7ej091ksKetrwAV+z1X0eFaWlKpVA1QAxAOD+rDGg/a0hX7NnyvKZ0EQGEizuTdW1m6opCFcyMHXP/6/dbvs364u//ZzXz5t88BMH/2FI6dMrrPXyOVSnHNPf9i5eZdfP4dUT52yuGvW3Pe7CmcfEQZJx4+ts9fX5IkSZIkSYrGgrf0xT6DvYP5ceDNUxcu6ziE9kxgE7AuIxUNYo3xBDvqm/d5blNJOXtyCwCYVruRmvpmGuOJA67fX8f12eCdxxxCZMJIzpk1iXEjR/TLa2yqbeTF7fXEEymOrBjZ6Zpp40o4/cjxFBcM9t8BSZIkSZIkKZsNaHo1deGyEmB628Mc4NCpC5cdC+xYt2jey1MXLvs6cOK6RfPOaFvzC1pnJ982deGyrwKVwELgK+sWzTvoEw6Hm8L8XMaGC/YJjZOhHNaWHsJRO9YxfedG/hU5mcL83AOu319ZuGDv+myQmxPi3k+e0q8f86TRRTx+zRk88OwrnDytrN9eR5IkSZIkSepvA93BPBt4pu2tCPhK2/vXtV2fCExrX7xu0bxaWjuWDwFWADcBNwDfGbiSh5b5sye/7rn2OczTajcyf/aUbtfve31Kl9eHo4EI1MMj8njf8ZMJhUJdrtuyq5H/+8dGUil/nyJJkiRJkqTBJ5RtwVU4HE7V19d3v3CIqm2IM3/J8n0O7jvzpSf47DO/Yk9BIUcs/xujS0Z0ub5dZUUJSxfMobQ4f0BqH4ye3VhLfVMLJx0x8J3Gz7+ym7ff+BcA/vTZ05g+vvNxGpIkSZIkSRrcQqFQQyqVGpaHww32GczqodLifJYumMMlVdMoC7fOXt4+sfUQuaLmRop3bOl2PUBOCH58wQlZHS7/oHoN7/zeo1zzm3/Tkkge9H5bdzdy9k2P8ZtnNhBPY78Z40sYXZzPiLwcVm8dvr8UkSRJkiRJ0tBlB/Mw1xhPMIIkz886nlQ8zqQbb2TUf7z9gOu37m7k1G88THNLkq+f8wY+cOKhA1jt4PLsxlre+b1HOaI8zE8/fiJTxhYf1H7f/sPzfP/h1YwszOPxa86gZET3I9BXbdnNoWOLs2oOtiRJkiRJ0nBzMB3MQSQ6EVgEvAMYCawFLo3Gguq26yFaz7G7GBgD/B34ZDQWPJfG3sXAscB49mtGjsaCe9Kpzw7mYa4wP5dQfj4jKisBaFy5ssv140cWcubMCgDueXpDv9c3mB09qZSfX3gSD/7XaQcdLgO8JTKet0Ur+NCbDksrXAaorBjZ5+FyYzzRp/tJkiRJkiSpfwSR6GjgMSAEzAOiwKeArR2WfQ64su35E9qu/TGIRLuctxpEom8DXgIeBe4B7u7wtjTdGtNLuTTkFc6cSeNzz3UbMAO8b9Yklv1rM0+ue5WXauo5rGxYjodJy6kzyl/3XGM80avQ9/jDxvCjC2Zn5MC+2oY4i6tXc/eKDdTUN1MWLuDc2ZO5rGp6Vo9BkSRJkiRJGiChUChU2eFxTSqVqknjvs8Bm6Ox4CMdnnux/Z227uUrgEXRWPDrtucuoDVkPh9Y0sXe3wWWAZ+PxoJN6X0Yr2fAnCUKj5oJS1s7mFOpFKFQ6IBr3zxjHOUlBWyva+Y3z2zkirdVHnBttqhtiPPdh1Zx7z82seMgA9quPved2bhzD39ZtY3Sonze8YaJPboXOj/Isaa+mSXVa3k4tjXrD3KUJEmSJEkaAHnA8x0efwW4No37zgYeCCLRu4C3AJuAHwE3RWNBCjgcmAA82H5DNBbsCSLRvwBz6Dpgngq8+2DCZXBERtYojEYBSOzYQcvWrV2uzc/N4d1vnATAPU9vzEjH7WBS2xDnzP+t5tbH1rGjvhl4LaCdv2Q5tQ3xLu9/6qUdbNy5p9evv/jh1Vxzz7+59dEXu1/c2f3Vq/cJlztataWOm6vX9Lo2SZIkSZIkpaUFOLLD2/fSvO8I4DJa5y6/ndau40XAJ9uuT2j7c8t+923pcO1AHmur5aDYwZwlRhx5JOTmQiJB43Mrya+o6HL9ObMmcetjL/LyjgaeeulVZk8dO0CVDj6Lq1ezdXdTp9faA9qFcyP7PL//SAqAk48o4wcfOr7H3cKnTi/n//6xibKSApLJFDk5PeuAXrqi61naS1esf139kiRJkiRJ6lOpVCq1qhf35QArorHgmrbHzwSR6AxaA+bvH2RNPwC+HUSihwD/BvbpoozGgqfTLVBZIKewkBFHHAFAY9D9HOajDhnFkRWtc8B//fTGfq1tsOsuoP3lky/v87h9JMWS6rV7w2WAx9fWpNXxvL8zohU8899nsuTDs3scLjfGE3u7rg+kpr7Zg/8kSZIkSZIGp83A/mFeABza9v4rbX/u301a0eHagdwNRIAfAo8DKzq8PZlugQbMWaRwZuuYjMaVQbdrQ6EQ58xqHZPxu39tytoAMp2AdmdDfJ/Pz//8fmWfjqQoyMshP7d3/6kW5ucyNlzQ5ZqycEGvDi2UJEmSJElSv+tsjEUl8FLb+y/SGiSf2X4xiEQLgTcDy7vZ+/Au3o5It0BHZGSRwpkzqf2/39K4svsOZoCzj5vENx6IsbuxhYeCrcw7pucHzA117QFtVyFzYX7OPgHtvc90PRd9oEdSzJ89mSXVa7u4PmXAapEkSZIkSVKP/C+wPIhEvwDcBRwHfBr4PEA0FqSCSPRG4PNBJBoDVgFfBOqAX3S1cTQWvNTV9XTZwZxFCmfOBKBl82ZaXn212/UVowo5ZXo5APc83fWYiOFs/uzJXV4//8RD977fGE/QnEh2ub43Iykamlv46fJ1XPSzFbxS29ijey+rmk5lRUmn1yorSri0alqP9pMkSZIkSdLAiMaCJ4GzgfOAZ4H/Ab4ELO6w7Ju0BtE30TreYiJwVjQW7O5u/yASPSaIRH8WRKIrgkj0ySAS/WkQiR7dkxrtYM4iI6LRve83rlxJySmndHvP+2ZN5q8vbOeRVdvYXtdEecmI/ixxULqsajoPx7Z2OvaisqKEz5xRufdxYX4uY4sL2NFw4I7n3oykyM0J8fX7AxrjSd5+1ATOPb7r0Ltdc0uS1dt2s3TBHG6uXsPSFev3zoU+/rAx3HrBCT0+dFCSJEmSJEkDJxoLlgHLurieAq5te0tbEIm+G7gH+Ctwf9vTp9J6kOA50VhwXzr7GDBnkdySEvIPO5T4Sy+nHTCfdVQF4YJc6psT/PYfm/j4qYcPQKWDS2lx/usC2rJwAfNnT+HSqmmvC2jnn9D3IylG5OVy1swJtCSTHFJamPZ9v/j7S1x730rOmlnBzR86noVzIyz6fcD6nXt436xJhsuSJEmSJEnZ66vA/0RjwZc7PhlEote1XTNg1usVzpxJ/KWXaQq6P+gPoLggj7lvmMjdT23gnmc2ZGXADK0h88K5ERbOjdAYT3TZgdxdx3NvR1L8vw8c1+N7HoptBWBUUT65OSEAFr4j2tUtkiRJkiRJ/a67fEUDohK4vZPnbwc+l+4mBsxZpjA6k933P0Djc+kd9AdwzqxJ3P3UBp7duIvnX9nNkRNG9mOFg1933/x62vHcn376sRP57T83ceLhYwfsNSVJkiRJkjpT2xBncfVq7l6xYW9ecu7syVxWNd1/ZZ0ZW4HjgdX7PX88sCXdTUKpVKovixr0wuFwqr6+PtNlZEzdo4+x/hOfAKByxZPklnR++FtHyWSKU7/xZzbVNrKg6giumWsHbE8M5t/ItSSS5OV61qckSZIkSepftQ1x5i9ZfsB/8b10wZxhHTKHQqGGVCoVznQdHQWR6JeAK4FvAcvbnj4FuAr4VjQW/E86+5gsZZnCma+Fw02xWFr35OSEeO+sSQDc+8xGEsns+qXEwerLcPnh2FYuuPUJPnnH012ua4wnury+ctMuPvzjv3Pyoj/Tkkj2WX2SJEmSJEmdWVy9utNwGWDVljpurl4zwBWJ1jnLXwEuBR5qe7sE+DLwtXQ3MWDOMnljx5I3cSIAjSvTH5Px3uMmA7BlVxPL12zvl9rUvW11TVSv2sbDz2+luaXzYPiFLbs5+esPcfMja2hq6TxoLhmRx19f2M623U08s35nn9bYXbgtSZIkSZKyz9IVG7q5vn6AKlG7aCxIRWPB/0ZjwWSgFCiNxoLJ0Vjw3WgsSLvD1BnMWagwGqVu8+YezWGePr6EN04ZzT/X7+Sepzfy5hnj+rFCHcip08s5+9hDOGV6OckDjLe58U8v8GpDnDv+/hIfP3Vqp2sOLSvmwlMP5+hJo6gcf/AztZ2hJEmSJEmSDqQxnmBHfXOXa2rqmwf1mNHhLhoLdvf2XgPmLFQ4cyZ1f/5zjzqYAd43axL/XL+TB559hevPbqFkhF8+A+2Q0UXc+J/Hdbnmy++eyZhwPiceXsaIvAN/U/7SO2f2SU2dzVCqqW9mSfVaHo5tHfYzlCRJkiRJUtfyc3MozM+hMX7gMZ1l4QLD5QEQRKL/AqqiseDVIBL9N3DATuVoLDgmnT1NCLNQ4czWYLFp7VqSjY3kFBamdd87jzmE63+3kj3xBA88+wrnHj+5P8tUL40fWchXz37DgL1eOjOUFs6NDFg9kiRJkiRpcPnivc92GS4DzJ89ZYCqyXq/Bpo6vH/Qh60ZMGehvQf9JRI0rVpF0TFp/TKCseEC3nLkeB5cuYV7nt5gwJxhe5oT5OTQZZdyOuqaWtjZ0MzkMcW9uj+dGUoGzJIkSZIkZa/3nzCFe55eT2F+HrV74q+7XllRwqVV0zJQWfaJxoKvdHj/2r7Y00P+slBeRQW5Y8cCPTvoD+CcWZMAeHxtDRt37unz2tS9RDLFh3/8d974lQd55PltADS3JPn0nc+wYt2OHu11w4PPc9x1D/LV3wW9qqUnM5QkSZIkSVJ2OnbKaB64oorqq0/nkqpplIULgNaxGJdUTXO8ZoYEkeifg0h0dCfPjwoi0T+nu48BcxYKhUJ7x2Q0ruxZsPiWyHhKi/JJpeDeZzb2R3nqRm5OiPqmFpoTSR59YTsAdz7xMr/95ybmL3mc9Tsa0t6rYlQh8USK5Wu2k0j2/F9EFObnMrbtfxQOxBlKkiRJkqTBqKfNUDZPda3j5+ef63fyjQdipFKvZQ2Hl4cZXVzAwrkRnvrSmcSu/w+e+tKZLJwbMVzOnNOBzoKdQuDN6W7iiIwsVRiNUv/ooz3uYB6Rl8u73jiRn//tZe55egMfP2UqRQV+GQ20BVXTaIwnOGV6OY3xBDMqSohOHMXMiaOYMjb9URdnzaxgVFE+p80oJzcn1Kta5s+ezJLqtV1cd4aSJEmSJA20xnhiSDf79Kb+dO6pbYizuHo1d6/YQE19M2XhAs6dPZnLqqZ3GnL2dH226ezz89bIeO775yYaW5KMLspnwQFGXwzlr8+hLohEZ3V4eEwQiXb8J/G5wNuBtDtLQx1/k5ANwuFwqr6+PtNlZNyuBx5g4xX/RSg/nyOffopQfvrfFP+yahsfufWJvY/95jrwOvsG/r7jJ3PBnKlMGl004LXMX7K804P+KitK/GcukiRJkjRAhnoY2pv6e3JPT39+HW4/7/Y0tO9ufVefn5IReYRH5PKzj5/EkRNG9qre4SYUCjWkUqlwpusACCLRJK8d7tdZx+Ee4FPRWHBrOvsZMGep5pdfZs1Zbwfg8P+7l8Ijj0zrvuH2zXUoGox/B7UNcW6uXsPSFeupqW+mtCif958whU+ePjT+T4wkSZIkDXUH+7Nipjuee1N/T+/5+v1Bl/8C94I5h/H2mROIJ1NUVY7rdv0lVdMG/aH2/dmx3d3n5yMnH8Z17zm6zz+moWqQBcyH0RosrwVOBLZ1uNwMbI3GgrRnwhgwZ6lUMsmqE08iWVfHxK99jdHnvDet+4bDN9ehrj/+DnY1xnn0he3MnjqG8SMLe3Rv7JVdHFkxklCo9Rdemf4/JZIkSZKUjXrzs+Jg6nhOp/6PnHwYa7bVkZ+bw5uOKOv2nnceM5GmliQlI/L43/cfy6zr/9jlQfUlI/Koa2qhtCiff375rG7Xl4ULeOpLZ6b3AWZAf3RsjyrKoyWZIj83Z8h/fgbaYAqY+5rDc7NUKCeHwmiUhiefbJ3DnGbAvHTFhm6urzdg7md9/XeQSKY47ZsPs7MhzlfPPpoPvemwtO8NNu9i7nf/yhsnl/LDj8ymYlSh4bIkSZIkZUA6Pyt+4s2Hk5+TQ2lxfqdhYk19M0uq1/JwbOuA/+vYdOqvGDWCr9y3kiPKw/z5qtO7veeR57dR19RCeUkBjfFEl2EoQF1TCwDxRJI9zS3drq+pbx7UTVaLq1d3GhYDrNpSx9fuD3jfrMmUlxRwxLiSbtef/6O/sXHnHs6bPYXPnlk55D8/ahVEonm0djEfyn4H/kVjwc/S2cOAOYsVzpzZGjAHQVrr0/lm7DeP/tUffwe5OSFOOnwsfwq2smnnnh7Vs+xfmwHYuSdOecmIvc8nkinWbKvjsLJiRuT5tSBJkiRJ/SndnxVvrl7Nj/+6jrlHT+DQsuIuw8Sbq9cMWANZuvW3/cNZEqlU2oHxf54whbHhAgrzcxkbLujynrHhAh6/5q17f47tbn1xQS7rauqJTBjVZR39pbuf/bsL4O99ZiN3Pbmed73xEL73geO6Xb96ax1NLUnWbqtL6/NZ1vZ51+AVRKIR4D7gcFpHZiRozYvjQBNgwKyuFc6MAtAYBKSSSUI5OV2v95tHxvXX38EX583km+e+kdKinv12+sqzKpkzrYw98QS5Oa3/S7+zoZlTFv2Z+uYEv750DscfNqZHe0qSJEmSeibdnxWf27gLgHEjRwyqf6Gcbv0fftNUPvymqXt//kznnkXvO2bv4/mzJ3c5UuO82VP2aZLqbn1Dc4J3/r9HueMTJ3HSEWWvu94fDXjdjTVpSSTZXNvIuJEjug3gm1qSAOTlhNIK7JtaknztvUdzzOTRQPefn/mzp/Two1MG3Ag8BRwLvNL2ZylwM/z/7N15fBN1+sDxT5reB6UttNwUetBwyCnKZQVEwaq7ouCqKKuC4u2KR3WX1ZVV+Hnsuut6sKDrgcfK4bUVFQW5RbkRE+5yCoVetE3Tpkl+f6QtbUkmM03SpuV5v168aGcm335zTTLPPPM8/EntIMoRRdGqhffuDYDDbKYy95Cq20wa0sXDetl5+Js/noOu8ZGag8sAOp2O4antGGtIql3WNjKUmHDnWDuOFmkeUwghhBBCCCGEdmqOFf996xAW3nEREwd1Vn11bFNRM399kK42uKz2NnXdk5lKelK0y23Tk6K5OzNF9fZd4iLo3Dac1MRoBtVJrCo2W5mzzMjg2cvJmPUVg2cvZ84yI8Vmq+Jc1agpazJv1QHyq5+/mrImk+at55MtRxk+dwW3vvUjYcFBxEeFKo4XHxXKL89cwXPX9qsN8itJiArlpou607dzLKD98RQB6ULgrwaTsQywA8EGk3EL8BjwktpBJMB8Hgvt0QNduLOhm8X4i6rbKO082kWHys6jCbSEHfjfbujPtw9nMnVYcnNPRQghhBBCCCHOC7cN70FMuOsL1WuOFduEhzAyrR0DusapCiaeKqnwx1RdqrTaSWrjuum8u2NdrcfHsZEhLLprODMyU0iovv8JUaHMyExxWXNaafuc+0fx7cOX8u9bhhCid4bXis1WrnpljdsAsLdBZk81ktfuO01eSQUHT5fxy69nPAbgJw/pSmRoMBGhzixrrQF7rY+nCEg6wFz98ymgc/XPR4FU1YM4HA4fzyuwRUVFOcrKypp7GgHj4A03YNm+g/g7bifp0UdV3abYbOX1VftZtOkI+WWVhAUHUVFlp014MGuzx9AmXHYg/tbwOUiICmXSkK7cnZnS6B14icXK97tPsfNYMU9eaVDc9rNtx9h9ooRbhyXTIdb1FwAhhBBCCCGEEE3n3Q25/PmzXeiAtpEhFJqtiseKc5YZFcsbXDeoC0u3HmV8nw7MmdiPtpH1A9K+LP+w/JeTTH93EwDjeiex5VCh6mNdb46Ptd4HT9s/9dkuG0PrFQAAIABJREFU3tmQ63b9jMwUr8qODJq93GNJkBmZKYw1JNKzfbTLRo410pOizwkCa92+IenJpUyn05kdDkdUc8+jLmOGYTXwd4PJ+Ikxw/ABkAA8B0wHLjCYjBcoDlBNajCf58J798ayfQeWX9RlMIPzDFX2hAyyJ2Rgsdo4U27lkhdWcsZSxYLVB3j48l5+nLGAc58DX+zAjb+WcP+HWwGYclF3uiVEutzO4XDwyop97Msr5WhhOf+8caDXf1sIIYQQQgghhHe6xkXSp1MbenWI4W+TB3g8VrwnM5WVpjy3wUQAhwN2nywhOswZPvJU/7exBnZryyXp7TFXVPH6zYMI1gepPtb15vhY67G0p+2XbvVfXWu1jRBvGda9dp41GcZqA/Bat29Igsst0rNATdD7T0AOsBI4DUxWO4hkMJ/nChct4sSsP6OPjSXthw3odDrPN3Kh5sxnZKie1Y+Npl10mI9nKvytymbn4jnfkZ4Uw5NXGmprKjVksdr453d7eX/jYd65fSgDurZ1u92u42dIjAmja7zrYLUQQgghhBBCCN9xOByUW21EhqrLJ1TK/o0K07N06zHaR4cxOiPR6+xWT+x2ByWWqhZbVsFitZEx6yuP25lmj29UILbEYiXzhe89ZjBvnjVOcY6+zNgW2gRiBrMrxgxDPFBoMBlVB40lwHyeK9+1i9zrrgcg9btvCenc2cMtXCssq+SS51dSUlHFHSN7MOuq3r6cpmgilVV2QoPVlWb39EEz/uXVmE6U8IfL0nnwsjRfTVEIIYQQQgghhB94OsbzVFKjMeUfbHZHvaZ9LZ2nEhYRIXqMs8drGvPX4nL+mmPEePwMYw2JzF9z0O223pbgEP4ViAFmY4ahA87GfkcbLO8CWA0m40k140iTv/NcWFoaBDvPbJZrKJPRUFxUKNNG9QTgvR8Ocbyo3CfzE01LbXAZPF/6UpMBffC06wYEQgghhBBCCCG8d6TATGlFldfjeDrGW7TJc/kHLb4znuQ3r64l93TrSQL01CRvdEZ7xfUWq+2cZfmllXy581cOnC6jU9sITU0NhVBhITDBxfIrgPfUDiIB5vNcUGioM8gMVBiNXo11x6gexEeFUlll55UVe30xPRFg9uWVovaqhwfHpvHDE2N5+XdSo1kIIYQQQgghGnIVTGyMJz/ZybA53/Hhj4d9Mp4rauv/qr1PZRVVPLp4Bz8fO8OfPv3ZF1MMCPdkproNAHeMDee5a/vV/l5stvL35Xs4WmhmzjIjg2cvJ2PWVwyevZxnc36h2GwFnMlb91yawtyJ/Zg6LJlFdw1nRmYKCVHOpos1jf28LVEizltDgNUulq+pXqeKNPkThPc2UGE0YtnV+AxmgOiwYO65NIW/5hj5eNNR7rwkhR7tAirzX6iwas8plm45Svf4yHoNG0+VVHDlP9fQs10Ur9w4kLSkGMVxpO6yEEIIIYQQQq3zpdarr5vknTxjYePBAiqr7MT5MbgYHqInPipUMcisA776+QS/Hei59GZUWDD/+N0Anvp8F3Mm9vO4fUuhpUne66v288aq/by6ch9V9rOJXPlllcxfc5BVe07VBo0fvSKj3t9obFNDIVwIBlw1Ugt3s9ztIOI8F27oTTFLsXhRIqPGlIu7s2DNQU6csfDyt3v4h2Svtjibcwv4bNtxusVH8odx6bWNH7/adYLKKjvHCsvp2DaimWcphBBCCCGEaOl8HWwNdK6a5OWXVTJv1QFWmvIalYGa1CactY+PZsnmY4zr3cHXU65n0pAuijWYHSiXXWwYDB2V1p7lf8hsVTWYQX0AOCw4CL2OesHluvacLOX1VfsVaypLcFn4wEbg7up/dd0L/KR2EGnyJzBv2cqhm24CIG3NaoLbK9cE8uSDjYd58pOd6HTw5QOjMHRs44tpiiay7UgRL32zm0t7JTJ1WHeC9c4vCA6Hg82HCjlcYGbiIOW6UjWKy63sOFqEDh0j09r5c9pCCCGEEEKIFsRVsLVGelJ0i7zcvzma5DUlpeesZ/sofjOgEw+MSatNUqqsslNUXsmbaw/WnkSIiwxh8oVdW+1JBK0G/OUbisqtbtcnRIWyeda4JpyR8KcAbfJ3MbAC2Fr9P8AYYCBwmcFkXK9mHAkwC+xmM7sHDwGHg87/eoU2l13m1XhWm53L/raKQ/lmLjMksWCq6pItopV5+vNdvL0+lwuT41g0Y3hzT0cIIYQQQggRILwNtgZKaQAtWdgDn/mGQnPLDiYWm62qyj8AzF1m5K21uVTa7OeMk5YYzeIZLe8kgi9ZrDYyZn3lcTvT7PEB8VoX3gvEADOAMcPQH3gUZ1AZnMHmFwwm43a1Y0iJDIHDaiWoTRvsxcUcu+9+TsTH03bitSRMn44+NlbzeCH6IB4el86DH23jW+NJthwuZFC3OD/MXAS6AV3bAmCutOFwOGrPZAshhBBCCCHOb4s2HfWw/sg5AeZAK6nhqeTFm7deSNcEZ28ai9WmGFyuua3awHlhWSWPLNrObSN6MCI1ocmOtdSWf8gvrWD+6gPY3OQ07s3zXP6htVNT1zohKlSCy8LvqgPJU7wZw31xHHFesBUXc2jKFOzFxWeXFRSQv+BNDk2Zgq3Oci2uvqATGR2cTeBe/Hq3T+Yqml6Vzc53xpP89X+/cKTArPn2l/dJYvtTl5PzwCgJLgshhBBCCCEAZ7BVKagGZ4OtAEcLzeSXVjBp3nrmrTpAfvVta4K5k+atp9hD8NYfXlu1z2W5CHDWzx31wkpOnrEAzmCipyZ8WoKJ7244xHemPO58bxNnLFXaJu4jSnNNiA4jKkw5p3HRpiO+nlKLM2mIcvnJSUO6NtFMxPnEmGGIr/uz0j+1Y0qA+TyXP38+FXv3uVxXsXcf+QsWNGrcoCAdMy/vBcD6/fms23e60XMUTa+wrJLLXlpF2h+Xccc7m1iw9iCT3tD+pS0yNJjYiPP3kichhBBCCCHEufLLKgnRKyeg1A223vrWj1w85zvFYO7rq/b7fJ6eeMrCBvjhQH7tz5MvVA4W/nZgZ9V/u3/XWAZ2a8uNQ7sF5DGXxWrzGPiuexLhfHVPZirpSdEu16UnRXN3ZkoTz0icJ04bMwyJNT8Dp1z8q1muigSYz3NFS5Z6tV7JZYZE+leXSHj+692cb/W+W6pis5Ub/r2BfadKqfuMnThT0WyZAUIIIYQQQojW40iBGau72gnVajI380osHDhV5nH7ps6GVZOFDTAq9Wyzc6VgYrBexw0aslUv7ZXI0ruH8/j4wCwxUVP+QYmUf3CWHFl013BmZKaQUP14JUSFMiMzpUU2uhQtxmigoM7PY1z8q1muitRgPo/ZLRZshYWK29gKCrBXVBAUFqZ5fJ1Ox2NX9OLmBRvZfqSIb415jEprd95/gAQ6T5d5aa2TVWSu5IcD+RwpKGf6JT19NU0hhBBCCCFEAFOqz3txzwRuH5HMlztPcKK6hERddTM320eHsfyhSxj38mrFv6elfrEvqK2fGx999li6JpjYsElevy6xPHZFL9Kry0yqpdPpCA0O3FKEk4Z0UWzkKOUfnNTWtRbCh6YCW4ASwAGsN5iMXtXa8TrAnJydE5I7N0tSGlugoPBw9HFxikFmfXx8o4LLNUaktmNochw/5hYy473N2ByOZm/EIJQ1ptmGks2HCpmxcAtBOrj54m5Ehnq325EPXCGEEEIIIQKTqyZ8Ewd1JkQfhMVq589X967d9s9X9+HBsennBFsnDenK3ZkptceKOp2OtA4xqpuhVdns6IN0LnvAaD2WaLi9xWpj/f7TjMlIAhoXQFUTTKyoslFWYXOZAbz3ZAmJMeEt4lj6nsxUVpryXCYwSfkH1+RYVzSRKcCTOAPMK4GOQJ43A2qK9CRn5zwAHMudm7Wk+vc3ganJ2Tn7gWty52ZJN7cWpu11E8lf8Kbiem8Um638Wn1G2lZdIqNuV1255COwaGm2ofaDr3/XtoTodRg6tuFUSQXdE7QHmAOtW7QQQgghhBCivmKzlUnz1tcLJuaXVTJ/zcHa3yf068CFyWd7RmnJ3FQbzH1nwyE+23aMey5NZXzfDpqPJdxtP75PB+77YCvHi8vJuX8UvTu18TqA6ur+Wm127vtgKwdOlfL+tIvpEBteu87hcPDwx9s5cKqUp67u47Gmc3Nzl7Hd8CSCEKLJ5QL3GzMM3wA6YJgxw+Ay+9RgMipfPlJNa6TnAeB2gOTsnEuAycBNwHXAS8BVGscTzSxh+nRKV61y2egvpGtXEqZN82r811bt40hBuct1jSm3IPxL7WVeWs6qtosO4+e/XEFYcOPOxLr7oionKYQQQgghhAgcSqX2APp2aoOhYxu36z0dY6gJ5lptdt5cc4DjxRa+353HsJ4Jmo4llI49Vhjz0OkgSKdjy+FCendq45cA6o6jRaw05VFld/DVz7/y+xE9atftOVHC3rwSLFY73RIiNY/dHKT8gxAB6VFgAfAEzhIZn7jZzgGoetNqDTB3BmpOP14NLMqdm/VxcnbOTmCNxrFEANDHxtJ94ULyFyygaMlSbAUFEBQEdjshHTsQ1Mb9FwA1fF1uQfifP+pkNTa4DL6vCS2EEEIIIYTwPU/Hfr8WW4gOa3y5PDXBXIfDwUuTBzBv9X6mjerp8VjiXyv3ct3gLoTqg+gWH6m4/d68Un47oBP3jUkjNfFsoz5fB1AHd49n3i2D2X6kiKnDk8/JqI6LDOHC5HgyNNZrDgQSXBYiMBhMxs+Az4wZhrY4m/31oSlLZABngETgCDAOeKF6uRUId3cjEdj0sbEkzpxJ4syZ2CsqKF21imMPPIj5x58oW7uO6FEjGzWuP8otCP8LtDpZcpJCCCGEEEKIwNZUx36egrk6nY5hKQkMS0kAPB9LfLzpaG0Jjy2zxnncfs3e07z8u4Fu1/vquHasIYmxhiSXGdWFZitr9p5m8rwNcjWnEMIrBpOxyJhhGA3s9bbJX5DG7b8B5idn5ywAUoFl1cv7cDazWbRgQWFhxIwbR8RA54dm3osv4rDZGjVWTbkFJVrLLQj/q8kMmJGZQkL185cQFcqMzJRGf4Epr7SxePNRZn36M6dLK1TfTssXVSGEEEIIIUTzaI5jP09jqTmWKC631v5sdzgC7thDzdWcQgihlTHDEF/n151AG2OGId7VP7Vjag0w3wusA9oD1+fOzSqoXj4I+FDjWCJA6XQ6Eh99FICK3bsp/vyLRo81aUgXD+sDuynB+aomM2DzrHGYZo9n86xxZE/IaPTZcQcOHlu8nfd+OMS2w0WqbycnKYQQQgghhAgMDQOrx4vKefjjbfx8rBgIvGM/NccS8VGhbHxyLGseG018ZGjAHXuouZpTCCEa4ZQxw5BY/fNp4JSLfzXLVdFUIiN3btYZ4H4Xy5/SMo4IfJGDBhIzbhwly5dz6h//oM2E8QSFa6+ColRuAeDS9PbeTlX4mS++QEWGBjOoWxzBeh0hwdrOa/mjJrQQQgghhBDCs4b1fxOiQrl+SBfuzkzhjnc2Yfz1DCeKLbw/7aKAK7UHno8lJg/pSlKbcNXbN+Wxh5ScFEL40RictZcBRvtiQJ3D4VC9cXJ2Tm/Aljs3a3f17+OAqcAu4PncuVkBf516VFSUo6ysrLmn0SJUHDzIgauuBpuN9g8/TLs7pzdqnGKztV4jhvioUKpsds5YqkhPiuaL+0d61QROtAwOhwOdTqf5dsVmKyOfX0GJ5dxyQOlJ0VJ3TAghhGgGEtBoeeQ5E1q5qv9bIz0pmrsuSeG5L408eFkaUy7qTlCQ7pxjv4ZN+ALtPjQ8ltC6vb8Nmr1cMcicEBXK5lnjmmw+Qgjv6HQ6s8PhiGruefiD1gDzD8DLuXOzPkrOzukK7Aa+By4A3sudm/WEX2bpQxJg1ubEM89Q+MGHBEVHk7L8G4Lj4rwar+aL7abcAibN24DDAXdfmsLj46VJm3DtRLGFYXO+wwFEhekpq7DVflHN6teBfl3aNvcUhRBCCM1aYrDPXSbjPZmpcrI3QMlz5l8t8X2sxZxlRsVs3hmZPblvTBrRYa4vjA6Ux0dr0DuQguSen4MUaXguRAsSiAFmY4ahN2AzmIy7q3+vl0xsMBlVJRNrDTAXAUNz52btSc7O+QNwTe7crNHJ2Tmjgf/kzs1K1ng/mpwEmLWpys9n/7jLsZvNxN16Cx2efNJnYz/3pZF/rz5AkA4W3z2cQd28C16L1mvL4ULe/+Ewz17bF3BeCvbAh1vZcriQ7x+5lO4JAbV/FkIIIVxqycG+QMvqa418HYyT58w/WvL7WKvWmD2r9X3W3EFyeR8L0boEaID5B+Blg8n4kTHDcE4yscFkVJVMrLXJnx6o+YQZC3xZ/fN+IEnjWKIFCE5IIGH6NAAKP/yIysOHfTb2w+PSSU2Mxu6ARz7eTnllwFdYEV7670+Hefi/21hhOqnpdoO6xfHS5P6Eh+gJD9HTPjqMwwVmHA74bNtxP81WCCGE8J2aIMG8VQfIrw7Y5JdVMm/VASbNW0+x2drMM1T22qp9bntq7DlZyuur9jfxjFqHYrOVOcuMDJ69nIxZXzF49nLmLDOqej00bLjWkDxnvtfS38daaKn/25JoDRY3dwZ2bGQIi+4azozMFBKqGxAmRIUyIzNFgstCCF/JALZU/3w9sNFgMl4J3ALcqHYQrQHmn4G7k7NzRuEMMH9Vvbwzzu6CohWKnzqV4PbtwWrl1Msv+2zc8BA9L03qjz5Ix4HTZTz/tclnY4vAtGTzMZZuPcbqPd7tLkKDg/jzVb15+7YLuW90qo9mJ4QQQvhPSw/2Ldp01MP6I000k9ajMcFKLQHppn7OWlqgsTFa+vtYDavNzltrD/L4kh3EVwc03UmICm32AOz5IDYyhOwJGWyeNQ7T7PFsnjWO7AkZElwWQviKT5KJtQaYHwem40yV/jB3btbO6uXXAD9qHEu0EEGRkbR74H4Azny5jPIdO3w2dv+ubbnnUmc34f+sy2XD/nyfjS0CzyXp7bjMkEjfzrEety2tqOLZnF84UmB2uf7q/p24tFciQUHaGwcKIYQQTa0lB2hbayZjc9MarPQUkD6Sb2b7kSLAN8+ZmufTmwzslqglv49dcfUcrzDl8cz/fuGzbce5qEe84u0nDenqr6kJNySgL4Twg5+Bu40ZBq+SiTUFmHPnZq0G2gPtcudm3V5n1Tzgbi1jiZal7bXXEprqDATnPf8CWmp3e3L/mDQMHdsA8Oji7ZRWVPlsbBFY7huTxoKpF3L94C4et/3ox8PMX3OQK15eTYmldR6kCCEChwTGhD+19ABteIheMhn9wFOw8q21B3n6812cKLYAngPSo15YyS1vbsRud6h6zgBmfry93u9aAsbnU7kIaPnv4xqenuPLeycxNDmeSYO7MPPydNKTol2Ok54Uzd2ZKU05dSGEEP5RL5nYYDI2KplYawYzuXOzbEB5cnZO3+TsnD7J2TnhuXOzcnPnZuVpHUv4n91i8ck4uuBgEh95BADzpk2UrvzeJ+OCs9zB3yb3J0Sv42hhOc99afTZ2KLligjV0y46lN8M6ERMuPvLv06VVPD35XtYaZJdkBBCm/Mt8040n9YQoJ00RPnk8ICubZtoJq2DmmBlpc3O2+tzqahyBiw9BaQBSiqqOFx99Zen5wwgPSmm9udis5XfvrZOdcD479/uafXlIupSG7SvtNndrvN38NnT+GpOCuh0OhZOu4gXJvUnNTFG6v8KIUQrZzAZa5OJDSZjo5OJdVoyUZOzc4KBOcB9QCigAyqAV4A/5s7NCvgjsqioKEdZWVlzT8OvbMXF5M+fT9HST7AVFKCPj6ftxGtJmD4dfazn0gTuOBwODk/9PeYffyQ0JYWen32Ko6qKoPBwn8z71ZX7eOHr3QC8c/tQMtPb+2Rc0XJZrDbKK23EKXyZv+0/P7Jy9ykuTI5j0YzhTTg7IURLdr53ZbdYbQEdzGyN5iwzMm/VAbfrZ2SmkD0howlnpI3Se0av02FzOHhpUn+uU3GVknAaNHu5YpA5LDiIoT3imXfLYIJ0OjJmfeV22xqb/jiWdjHO7+ZKz1laYjSPXN6L/l3b0iHWuf1zXxr592r3r9GucRF8/+ho9NXlyfo9/TUlFvdXHiZEhbJ51jiPc25JPL2PDR1iWPbQJbW/l1faqKyy89qqfSzedJT8skoSokK5fkgX7slM9cnnTLHZqnp8b/dD8tkhhBDe0el0ZofDEdXc81BizDBEACOAvQaT8ZDa22nNYH4emALMANKBNJzR7FtwBp5FM7MVF3NoyhTyF7yJraDAuayggPwFb3JoyhRsxcWNHlun05H4qDOLuXL/fvZcdDG7Bwxkz/AR5L34oldjA9x1SU/6V2e/PL54B8Xl1oC/xExot2znr9z13iZVmerhIXrF4DLArcOTiQkLZkDXtlQpZIwIIURd50OjpoYkY7t53XJxd7eXmqcmBv6l5k9/sYtr+ndm+qie9TIZbxuRTEbHGGLCg0lzc/+Ek8PhYPHmo7xYnVDhKcP4thE9eO+Oi4gMDVadBV8TXAZnYzB32aeLZwznir4daoPL4CxPpuRIYTmnSysAZ6BRKbgMLaNchFb3ZKa6fR+nJUbzxpTB9Zbd/s5PXDznO7+VEVFbpiTvjIXp725ivsIJBPBcQ1qCy0II0foYMwxvGzMM91T/HIqzLMY3wG5jhmGC2nGCNf7dm4Dbc+dmfVln2f7k7JxTwALgEY3jCR/Lnz+fir37XK6r2LuP/AULSJw5s9Hjh3brRlB0NPbSUuzVmeA1AezSVavovnBho7Okg/VBvDSpP1f+YzUnzlgY+uy3VFTZfX6WXzSvPSdL+XrXSVITo3nySsM56/NKLLSLClPdvC8zrT0bnhxLdJjW3ZkQ4nymplFTIGeTauUqk7EmCLHSlNfiMrZbWhbdiWILV/5jDVf178jI1HZ8tu04+WWVxEeF0rltOEVmKyHBgdu0dvWeU3yy9RgAb0wZzB+zDPWeg+JyK0cLzfTp1Pgr5VoCra+7htsv3XKMRxZtR6eDy3oncU9mKitNeW6vpGh40mHSkC6K2aeuGq7FRoaQPSGD7AkZivO3WG2c8RAwBqisLtdRE/BWysCODgtuUe9TJa+u3MeI1HYM6NqWRXcN5/VV+1m06UhtxvCkIV25OzOl3n7UdOKMYgPzmpOZ3mQMqzlZmj0hgzYRIXxnPIndw8XLNScFWsvzJoQQrYUxw/A08FSDxScNJmOH6vW66vV3AnHARuBeg8m4S8XwVwD/rP75GiAG6ADcDjwNLFMzR60ZzLGAq5Se/YAUXgsARUuWerXek/z587GXuv4SUxPA9kb76DBiI5wZFhVVzmzU1tws5Hw0LCWBa/p3YspF3c5pFulwOJjx3mYu+/sqvt+trqZyUJBOgstCCE1aS6MmLVpDxnZTZ2D78vl/Y9V+zliq+HLnCe4fk8bmWeMwzR7PB9MvwvhrCUcKy/nPulyf/T1fy+gQw+QhXRjSPY4r+iQB9TMZYyNC6gWXHQ4Hmw8VNPk8/UHr605p+6v7dyI1MZrLDEm0jwlTzDB2ddJHKXtWTcM1paCh2gzprvFnr+r1lIGddUFHxfUtxSdbj/LC17uZPG8DWw4X1gbta97Hm2eNI3tCxjnPV3pijMfvqB//dG7WuJrXnMPhwHTiDO9tUL5yuSYjOTxEz52XpBAZqhw4DvRa8EIIcZ7bDXSs869fnXWPATOB+4ELgTxguTHDENNwEBfiqrcHGA8sMZiMecBHQG+1k9MaldkOPADc22D5g9XrRDOyWyzYCgsVt7EVFGCvqCAoLKxRf0NNANubDOnXVu3jVPWldw2pOcsvAt/QHvEM7RHvcp3pRAlbDhcBEKTTnsnlcDg4ccZCx9gIr+YohGjdwoKD0AfpsCmkcrW2g+yWnrHdVBnYWmqZapE9IYO4yFA6xIbVln4KD9GT0aEN949Jw+ZwMG1UD6/n7y+JbcJ5/vr+VFbZ0Xn4fHY4HPw1x8ibaw8y+zd9uGVYMtDyss5B++tOzfZLZtS/jdoM45pt1WbPNobWDGmlDOyucRE8OeHcK9W80VyvoYwObegSF0GXuAj6da6fpa80n0qbndIK5azwArO19n79WlzOzmPFvPDVbvbmnfsa+s6YV/v6cThg8hsbMFcqnwirm5GcPSEDBw7NWfBCCCECRpXBZDzRcGF19vJDwFyDybiketlUnEHjm3A261NyAuhrzDD8ijOb+c7q5dGA6kwOrQHmx4Avk7NzLgN+qF52MdAJUF2XQ/hHUHg4+rg4xSCzPj6+0cHlpghgt/QDYOEdQ8c2fHHfSP634zij0tppuu3Px4p58pOdHC4wsyF7LBEeMjSEEOcvnU7H1Rd05NNtx91u05oOsrVkbAdqAFDtZeDe8GcQOzxEz4OXpblc5255IAoN9nzxY0WVne1HnCeL1+3L52hhOYs3+6e5mb+ped09MDaVbUeKKDZb2Xy40KvXqZr3n5aAtFZaS3aoDXjb7Q5e+GY3t1zcnU5ttSUB+OukjxaGjm34/L6RAITo1V8ArKaMSN2Tmf/b/ivPKvQo2Zd39jUUFKRjaI8EvjOeRKnqRcOTpVqfYyGEED6n0+l06XV+z3c4HO7rKdXX05hhOA5U4CyB8aTBZDwA9MBZ0uKbmg0NJmO5McOwGhiO5wDzW8B/geOADfiuevlFgEnl3LSVyMidm7UaZ3O/xTgj2dHAIpwR7ge0jCX8o+11E71ar6QmgK3EmwD2+XjJsjhXvy6xPHGlwWOGVEMJ0aH8cvwMRWYr3/xyzkk9TeQ1Jlobra/plv4eaDj/iiobr32/j+Lysyfg/3JNX9ISG3+pubdzakpqL30P1OAyqDsB7S1/lBFpWApKDWsANaz9YONhtlUHi9UKD9Hz9u1DuffSFPafKmXeav80N2sKal53x4vKuWn+Ru5+f0uTvE7r8vV7VmvJjppyDFq8AAAgAElEQVTbeCoX8dLy3bz+/X5+8+o68s5Yapd72i+qbWCnpLH7XqvNXu/9Gx8V6nE/6oqnMiJ1T2buP+V6/1NX3dfQS5P7M/2SnqrHh8Y9x0IIIXwqGGepi5p/96u83Ubg9zhLWEzHGVBeb8wwJFT/DHCywW1O1lnnlsFkfAZnveV/AyMNJmNNYK4K+D+V89OcwUzu3KzjwB/rLkvOzukPXKd1LOF7CdOnU7pqlctGf2FpqcRPnYq9rIygqCgXt/as7XUTyV/wpuL6xtJ6ll+0XJsPFfL35Xs4ecbC8oczfTJmx9gIsidk0KdTLBf3dF2CQ0kgZMgI4UtaX9Mt/T3gbv53XZLC1Ld+ZOexYg6dNvN/118AOA+yF8+on3kXqg/ihgu78Mjl59bS9OWcmuMx9XTpu9arRnxNKRMzr8TSJBnYvr6KKr+0gpsXbOT+MWlc2a+DqhOny3b+yl9zjLz1+wvp1UFNyTz/OZxv5unPd1Fps/PqTYM01dONDgumyuGod5l/XS2h7JnaxIeaK6bCg4PqncRyt30gXykA3mVIu9t2ULc4okL1jOmVSGhwEHOWGVXtFxt75YK3+16Hw8ETS3disdp44fr+Xl0VpyVj+Olr+vDRT8onIeq+hmIjQrj30lS+360tI9mfWfBCCCE8qgL61vldVfaywWSs12jPmGH4ATgATOVshYlGqymt0WDZO1rGkM5YrYw+NpbuCxeSv2ABRUuWYisoQB8fT9vrJhL/+99z4um/UHX6NF3/PQ99jPYDF6UAtj4hgYRp07yav6cD4Cv7tY5mIQLW7jsNwIliC1V2O6+u3M+dl/SkR7vGnfwAmDZKOYvDnaaq7SlEU/FH3dBAfg94mv+V/Try8/FiosKCsdsdBAU5A33+PMgOtMdUKcihD9Jxp4csOH9QGwRS08g13ssT0P4oI/LiN3swnSjh8SU7uLhnPAnRyld4Waw2/ppj5FhROU99/jMf3TlM9fz9objcSveESIrKrVzaq73m27f0smdqEx86xUZgmj2e8BA9g2Yvb1WJEr6a61hDEp/fP5I24SFMnrfB7X7x47uGsS+vlIOnyxjaI17Va2hC3w4ktQknqU0YOp3O632vxWpjw/58Fm92/u2hPeK5tbqOeGNoqZvdmGQbb+tyt6TXoxBCtBIOh8Oxx9tBDCZjqTHDsAtIAz6tXpwE1O0em4SzvrJHxgxDMDAU6AbUu2THYDK+q2YMCTC3QvrYWBJnziRx5sx69ZBL16ylZPlyAM7k5BD3u981auyGAWxdWBiOigpsZWXYS0vRx8Z6HsgNpQNggJ1Hi7Da7Jrqn4nA06dTG24c2o0BXWOJCNXz9+X7+fDHw3xrPMmG7DEEN/Hz2xS1PeuSbBHhb55f0/vIrtN8qanfA77maf6X9rLx+b0j6dfF/edTw/ek3e5Ap0NzuR61c/L0mPpqP3Gi2EJSmzC3QYjL+yRx/eAu9O7U+M/uxlAKAn380xH+eKWB66sv7Y4MDWZg17ZsVSjVMNnLmtnhIXraRoRQpJCBqjU4eMfIHhwtNHNJWnuPweWaOcyZ2I83Vu1nzsR+Hrf3t35dYln24Chy881EqQjy1+WLgH1zf1au2XuKmLBgxfsxaUhXdDpd7Ty1Nsk7n6S0j2bOMqPifvGNVQdYvPkIp0sreeY3fVS9hu56bxMnzlQw+7d9ueXi7o3a97o62XVRj3iiw4KZclH3xt3hOrSczGzMa0gykoUQ4vxjzDCEAxnASuAgzkDyOOCnOutHAY+qGCsD+AJnLWcdzjrMwTgb/FUAqgLMEqVr5erWQ44eNZIOz/yF+Dtup+0NNzR6zJoAdvr6dfTavo201avQx8eDxcKJOXO8mq+7umBX9EkCYNvRYv5vmeoa4yJA1RxE33BhN8KCg0hpH0WHNuFMHdbdJ8HlX4vL+b+vTOzLK1G1fVPUTCw2W5mzzMjg2cvJmPUVg2cvZ84yY4uoQSlaHk+v6TdWHaDnEznY7Q5V2/u6bqiveZr/ks1HFYPLDZlOnGHSvA189XPj67k35jH19X4i93QZE/6xmj99+jNVNrvLWqlzJl7A4O5nywpZbXav7rdaSkGgQrOVF7/ZXW/Z27cNJT3Jc81sq83OBxsPu6xj3LAW6+ZDhew+cfZz4oahysG/iYM6K65vKDUxmndvH8odI3uovs0l6e15f9pFdE9o/NU8vhSsDyLVTa1yJWrrfn+96wQvfr2bvBJnXd5A+aw8UmDmjrc3cajATGyE6wxQV+UH7slMVfU6PV+p2S/2aBdFXGQIdrvD42soLjKEvJIKAHolxaj6G+//cIhtR4ooq6gC3Nd53niwgCOFZkosVarum1qegr/evoYkuCyEEK2TMcPwojHDkGnMMPQwZhguwtkbLwp4x2AyOoCXgceNGYaJxgxDX+BtoBT4QMXwLwObgVjADBiAIcA2NJRDVpWOkJyd87mHTdqo/YPJ2Tn34IygdwR2AQ/lzs1ao7D9TcBjOJsLngG+BR7JnZvl/6OfVihu8uRzljkcjtoMLbvFQlB4uOrxgsLCICyMxEcf5dcnnqD02+8oWbmSmNGjGz1Hd2fh//q/X1iw9iAL1h5kSHIc4/tKuYyWylWmyMRBnbl+sPeZPQ6Hgxv//QO5+WaKzFaeurq32y/b5soqgnQ6v9f2DLRL5UXrpiZzEMDhgKAgnV9KAzQlf8x/7jITmw8VcqywnNEZiZrvt9o5/fbVtQxPacdj4zP8sp94d8MhCs1Wvth+nBmZKXSNj6xd5+o+2e0OHlu8g0+2HuO+0anMvDy99vuBr59/T0GgonIrNrsDfZ1yJmouA//36gO88PVuPvzxMItmDKPCandZhiMIHa+v2s/Abm1ZMmM4QUE6xauouidEct/oNM33U6fToTUJvm7WfE2TscZm0jfGjqNFJLeLok24d59LnjIxrx/chVdW7GNfXilHCs08c03fgPms7BofyYOXpbFo0xFemzKIz7f9qqr8gLflCloztfvFFTMziY10BpZ/PWNRfA3dcGE3Hhybxu6TJfRKilH1N0oqqvjtq+t49/ahXJLePuCu4JHXkBBCCDe6AB8C7YBTOOsuX2wwGQ9Vr38eiABeBeJwNgW83GAyqsm6uxDINJiMZcYMgx0INpiMW4wZhseAV4AL1ExQ7fVunopO5+NMyVaUnJ1zA/AP4B5gbfX/y5Kzc3rnzs067GL7EcB7wCM4a4okAa8B7wNjVc5dKDBv3cqJvzxDxMABlHz9zdmazROvJWH6dNXlLmJ/+xuKliymfNNmTj77HFHDhmkKVLtT92D28QkZbDtSxKZDhTy6aAe9OrTxql6vaB7ugijz1xxk1Z5TXh9A6nQ6Jg7qwqsr9/Hp1mN8+OPhc+p6bj9SxAtf76bEYuXTe0d4rHcXExbsVWAl0A5eROumpoZjTHgw/7xxoOrtm7tuqKvgps3u4JtdJzCeKPH5/P98VW+OFGziT1nuT1ApUfOYhgUHse1IMTHVQTx/7Cf+mGUgWK9jdK/EesFldypt9to5V9kdnCmv8kuTQjVBIIvVjtVmRx9Uv9aop8vAT5c6sxn7d42lwmp3G7Ds3DYCAHOFjVOlFSS1CVcI7HTh7jr32Wqzo9fpamt511VcbuUvX+ziobHpdEvw/JgrOVJg5slPdnL1BZ2YfGHXJjnJY66sYvq7m7DaHPxtcn8u7ZXY6LE8NTe7fWQP0MHJMxZuH9Ej4D4r7x2dytThyUSHBdO7Y6zq8gNSrsA1tZ81NcFlUNcgLyJUz4CubWuXe/obNbrEOfcBgVgrXF5DQgghGjKYjIo1bquzmJ+u/qeVDmfmMjiD152B3cBRIFX1IDWZEU0hOTtnI7Ajd27W9DrL9gKLc+dmPeFi+0eA+3PnZnWvs+w24JXcuVnar9cDoqKiHGVlZY25aatjN5vZO2Ys9iLXNQ3D0lLpvnCh6iCzZfceDk6cCDYb7e65m/YPPODL6QLOWpJZ/1xDflklGR1i+OSeEV51dhZNb84yo2I2yozMFK++yBebrVz3xnr2uehcn54UzaK7hrPr12Jumr8RgMUzhrHceFJxTgDX9O/EP343QHMWmc3uoP9fvqG0wv0llglRoWyeNU7TuEIo0fo+87T9tJE9+NNVvX06R088NYD7bNsxHvxoG0E6Z/3dj35yX8ajMfuVus0AtXI4HMz9yqT4mGb160iXuAh6tIvid0O7qWoO1hT7icoqO59sPcoVfTqc04yrRs2+1Jsg88BnvqFQoeyBN/d3w/58+nWJ5ZUVexWfg/F9knjlpkFu+zq4CuzY7Q4e/ngbVXYHL03uT1hw/fVPf76Lt9fnEhsRwvrsMZprF9f1wIdb+Xz7cUL1OqLCgik0W30W5HfFYrWx5XAht/3nJ+wOB9/8IdPrE/nFZqvHTMzyShsRoeoa5PnzPTB/9QHSkqK9CqoLZY35DqjmNaTtb/Rk2qiexEWGYrXZyZj1lcd51zRxFEIIIbyl0+nMDocjoDIljRmG1cDfDSbjJ8YMwwdAAvAcMB24wGAyqspgbrIazMnZOaHAYOCbBqu+AYa7udk6oGNyds7Vydk5uuTsnHbA74AvtfxtnU6XoNPp0nU6XXpTBtQDXVBkJFFDh7pdX7F3H/kLFqgeL7xXOvG33gpA/vwFVBz0mNSuWYfYcP5540B0OjCdKGHWZz8jz2nL4u9ar6+t2ucyuAxnM6CG9Uzgdxd2Zd4tgxnYLU6x3l1k9QmMlPbRboPLDet61l0++sWVisFlOHv5vhC+orWGo9L2AHvzSpp0X+uuJua8VQeYNG89xWYr4/t2oEObcEaktmPSkC4+r3va2ODyhv35THpjAzcP7aY4p+eu7ccTVxr43dBuqi8dP1zg/gR5zT7kzbUH2X/K9T5QjdDgIG64sBuvr9rvMZtUScN9WllFVb15Tb5QuSSSN83QhqUkEB0W7PHz5qfcQsWmwa4CSl/tOsGn247zvx2/8unWY/XWWaw2BnZrS/uYMO4Y2cOr4DLA/WNSCdHrqLQ5aoPxDd8H3mpY8/j+D7Zy/eAuPPObPj65SsxV3e/sCRn1AoMRoXpNpW58qWa8t9cd5Nkvjdz13mZ2HHXfTFJ4pzH1hdW8hrT9jVTaRYehD9KprhUuwWUhhBCt3LM4s5gB/gR0w9k88HJAdeZoUzb5awfogZMNlp8EOri6Qe7crA04A8rvA5U4U7V1wFSNf/t+nOndu6uqfNuooaUz//ST4vqiJUs1jdfu3nsJTkrCYbVycvZsvwQkRqS24+HL0gFYvPkoHwd48ylxVlMcQKoJYOt0OuZedwFX9OmAPkjntrnkjMwU1meP4Z83DuSe0WcPehwOB4fzy1w2IjpV3WwGnMGJvp09XwEgBy/Cl9774RCVNrvb17SrzFN374FhPeMJCdJx80Xdm7QGrJpL5cOC9Xz10Cjeu+MiBneP13R/tTBXVjHnSyP//emcSl7nKCirZNo7P7HpUCGzc4yq56QmyAFw2UurWWE6+zWqYXCwz1NfMft/v3Dtq+vYc1Jdk1N3fNmk8H87jjP6xe+5e+Fmqqqb7/m7GZq/Pm8m9O3AA2PTmDykC5OHdD3nPj/zxS9cdUFHbvAQQFdj8ZajWG2uv0epCfJ74u5EzvsbD/Ofdbk+b6yn9DnXlIE+V6/TvXmldGwTzrCUBNISY7z+G8I1pe9bavbVap5/rX9j0pAuiuN5c7JLCCGEaAkMJuPXBpNxafXPBwwmowFnDDfJYDJ+r3acJiuRkZyd0wk4BmTmzs1aXWf5n4Gbc+dm9XJxm97AcpwdDb/G2RjwBWBb7tysW9X+bZ1Ol4AzxZuIiIjdZrPZwy3OD3aLhd0DBnrcrtf2bc5mfiqd+eprjj30EACd//432kyY0Og5umO3O7j9nZ/4fvcpQoOD+OSe4fTpFCt1yloAf14Ca7HafHKpo6fX0cebDpO9ZCd2F7vP6LBg1j0+pvYAZl9eCfNWHWDRZvfBGm/LgghR46ufTzBj4WbaRYfy37uGkdLeGcDTum+su/2h/DK6JzTtVVze7id8+Vkw69Ofee+HQ8RGhPDdzEzaRSt/Hn6y9Sh/W76HhXdcVO9x8zQnT5d1hwUHEaIPYl32GGIjQlzWs68REaJn9WOjaR+j/rO7LrX70lGp7RicHMdDl6UrzqdrfARHCsoJ0ev46M5hDO4eB2i/9F0rf37eOBzOGtXu7rMvyoj4u2SEv0tW+Xo+mentefu2C7062aX0Ou3RLoqP7xrW6PeN0K4pvrd7+htKrwlfvI+FEEKIugKxRIaveHftnjanARvORn11JQEn3NzmCeDH3LlZL1T/viM5O6cMWJOcnfNk7tws5fSaag6HI5/qRoVRUa3yeWyUoPBw9HFx2AoL3W6jj4/XFFwGiLnicqJGjqRs7VpOPjeHqFGj0Ec3qmS2W0FBOv4+eQBXvbKWY0Xl/O7fPxAcpPN7fULhPU9d5b3JFPFVszKl9Ta7g2dzTC6DywClFVU8/7WJZ6/tB0BqYgx/yurN9qNFik1qhPCFSpudyFA9yQlRdK/T1E3rAXzd7RsGl3/KLWBI9zi/ZTRryTx1d798GbC4b0wqOTt/5cahXYkK9fy16dqBXZjQt+M5c/A0J0/NrN6+bShHC8uJjfDcFLDcauPNtQcbHRxUsy+NCNGzZt9pTpVW8NBl6YrzOVJQzojUBJ67tl+915O/G1n58/NGp9P5tSmdL94Hnh7TQGtupvQeiAkPZtWeUzz88XZemtS/0SVslJ6zg6fLvHrfCO2aIilETVNG1809fXeySwghhAg0xgzD52q3NZiM16jZrslKZOTOzaoENgMNUy3GAevd3CwSZ1C6rprfm7K8R6vV9rqJXq13RafT0eFPf0QXEkLVqVOcfuVfjZ2eorioUP7vOmet8RJLld/qEwrf8vdl0f6+1FEfpMPTYe2yn+ufM/P2ktBAI/Wifc9Xj+k1/Tvx5QOjePl3AwhWqC3bWG+tPcikNzbw3JdGv9VkDrSamEltwlnz2GgevSKjXlPZmuds8eajLG5whUJj5uZpP9GpbQRDe8TXbu/vevae9qWjM9pzw5CujO/bQdV8TL+WKGbC++P59PfnjT+fAy3vg9OlFdirz3q6K1NS9/uQxWqjxGJtlprHSty9B+68pCeZ6e0B6BYf2ejgMvj/fSNaJq11noUQQohWIF/DP1WaMoMZ4G/Ae8nZOT/ibOA3A+gEvAGQnJ3zLkCd8hdfAPOTs3Pu5myJjJeBLblzszwXQxQeJUyfTumqVVTs3XfOurC0VBKmTWvUuKHJySRMn87p116jYOFCYideS3ivc6qgeGS3WAgKD3e7fs2+U27XeZs9JPzD35kinrIAfVHXs6hc+cRFgYusMleZeg6Hgw0H8unbOZY24YF9EFNstvLaqn0s3nS09jmTKwW846/HNNkHjblccTgc7DxWDMDWw0VUVNlrX+O+zj69JL0dn2497nZ9U9fErGnW1vA5iw4LprSiCh0QHxXCmIyGF2lpozaj1xfZrZ542pfOufaC2tdpU8ynMfz5edMU91ltBva972/hVEkFfxiXxisr6mfo1px0X2nK4793DuOvOUa+3nWCFyf198kVP77m7j3gcDi46oKOXNHHZdsWVQL1dSoCizz3QgghzgcGk/E2X4/ZpAHm3LlZ/03OzknA2ZWwI/AzcGXu3KxD1Zt0a7D928nZOTHAfcBLQDGwAni86WbduuljY+m+cCH5CxZQtGQptoIC9PHxtL1uIgnTpqGPjaV8xw5Kvv2O9g89iC5IfUZcwp3TKf7iC6xHjnDi6b/Q/f2FOCorFQPGALbiYvLnz6do6Sdn5zPxWhKmT0cfW79hWqBd3inU8edl0f4OYPuiDEdNcHnqf35i9Z5TzByXzv1j07yalz+5qk9YN2jRErOwm5svH9OdR4v54UA+d4zs4VVmnyc6nY4Xrr+AlPZR3D6yBxVWO3//do/PA+QWq40f9rs/Ud5cZWVcPWelFc7GwWHBQaS1921jMDUN0fwZHNSyL22K+TSWvz5vmuI+qzlhui+vhI0HCwDI2XlCsWTHvNUHOFpoprSiiv/tOO7XEiK+UPex0+l0jO/bsd76tXtPExcVQp9Ozu+Grp7fKpudz7cf5+tdJ3j95sEB+zoVQgghhGguxgxDByDYYDIebbC8C2A1mIwnXd+yviZr8hcooqKiHGVlZc09jYBlr6ioV3O58ugxcq+/HltREfG33UbS449pGq909WqO3HkXAEFRUdjLyhQDxrbiYg5NmeI2o7r7woW1t/FVQzfRuvkjE8lXjZGe+eIX3lp3kEvS2/OOl42L/CnQGkG1Br56TM2VVVz1z7UcOF3GjUO7MmfiBb6cplv+boq0cncejy/eQWZ6e1aY8gKiJmagvQ+aej7eNilsjfuJprjPahoh/nysmE+3HmPplmMUmJWDp3Mm9uNYUTlX9utIeLC+xTY3+/lYMTfM24DD4WCsIYn1+/NdnujadqSI3766DoDXbh7E9qNF593rVAghhBCBIxCb/BkzDN8C/zWYjPMbLL8DuMFgMl6uZpymLpEhAlzDhn4hHZKIHj2akhUrXNZj9lTCIqJ/f3SRkTjMZuzVgX1bQQH5C96kZMUKkj/8sF6QOX/+fJfBZYCKvfvIX7CAxJkzgcDOmBKBw191PX1RhmNGZk9GpbXj0l7tAza4DHKlQGN4CsZ9/JNynU+1j6ndAf27tuVQgZnJTZht6M/mZgCjeyWy5vHRhAX7pwRHYwTa+8Df5YAa8rZJYWtsZtoU91lNBnbfzrGkJkazYO1BxbHyyyq5JL19vTFaanMznQ4iQvUUmq18sePX2uUNrwQZ0LUtmentCQsOIqV9NCNS2p13r1MhhBBCCA+GAPe6WL4GeEHtIJLBLDxyOBxYjx0jtIuz4Y+WEhYnnn2WwvcWuh077tZb6PDkk7W/77noYmzFxW6318fHk75+Xe3vnrKHRqQm8P60iz3eRyG0UpNV1hoUmSsZ8Mxyj9u19isF1AQ41dRUdjgcTH93E98a8zz+TaXHtOF89pwsIT3JtyUalAyavdzjyb3Nsxr29FWWV2IhMUa5hFJzCdQrZgJtPxRo82kKgXSfvX1fBsKJHC3++MlO3t/oviVLTTZyZZWd0OCzJd4C6TkTQgghxPklQDOYS4HhBpNxR4PlFwAbDCajqvlKgFloYisuJvfmKVTu81zCAsDUfwCOigr3A+r1pK74jpCkJOwWC7sHDPQ4h17bt9VmWitdpl3jD5el88DY1IDOEBUtm68OyqtsdoL16uuc+4q7+W88kM8DH20l70wFSp8UjQkmNjdfBYzrbutuX9Q1PoL/3Teq9jbT3tnEtx7KWNV9TGvmGiiNFv0RbN17soTfvrqOqcOTmXl5L/R+rCXdWP4IqvtSoAUHA20+TaG57/P5VqbEF+/J5n7OhBBCCHF+CdAA83fAHoPJeHeD5fOAXgaT8VI140iJDKFJ/vz5LoPL4Cxh8etTT5H05JOEJCZit1iUg8sANhvmLVuInTCBoPBw9HFx2AoL3W6uj4+vV8bDXROi3w7szK5jxfxwsIC/f7uH4nIrf8oy+LUBljh/eXtwWlxuZf7qA3yy9RjLHhpFm3DvgoW+Cp4mt4ui0Gz1+PeauxGU2gCBtwFjpSZ8//huj9sTXUcKyuuVjLh9RDIOh4PvTO6zmGse090nSrj+9fWM79eBrYeL2JfX/I0W1ZQnig4LJkTDyZLXv99PWaWNRZuPcvvIHrSLDvN8oybWkhqiBYJAm09TaO77fD6VKbFYbYr7IHDuIz19PjT3cyaEEEIIEQD+CKyozlheUb1sDDAQuEztIE2fKidatKIlSxXXl3z1NeVbtmgas3zrNhw2G4DLOs91RVxwbgOrmvqEm2eNwzR7PJtnjWPWVb15546hjO/TAYC31h3k8SU7qLLZNc1NiKZQYbUxf80BjhWV8+763EaNUWy2MmeZkcGzl5Mx6ysGz17OnGVGil0EiGuCp/NWHSC/+gC9Jlh51b/W1N4mqU04L1x/AV/cP4L0pGiXf7cmaPGvFXvZe7KkUXNvyGK1edxGy/2t2d7dfZ40b/05t1NTY7jG/NUHeGtdruJ8F206W3N5eGo7/jZ5gMfHFODDHw9TUlHFsp0n6gWXlebTFCYN6aK4vrSiisnzNmCurFI13pzr+jF1WHdeu3lQQAaXwRm8U/OcCdFcak66z8hMISEqFHBm8c7ITAnohn2NUXOiS4n04RBCCCGE8MxgMv4ADAMOAhOr/x0EhhlMxvVqx5ESGUI1tSUsOjzzDHGTJwFgGjAQh8Xi8TZh6ekkPjKT8Asu4PAtt7hs9KeLiCDlq2WEJCWpnnOVzU720p0s3uxszjS+Twf+ceOA2sZRQgSKF742YXfA9FE9PR40N6RUniE9KfqcwEJjLqNWqln5+Y7jzPr0Z2LCgllyz/BG1QL2VTkKV/dX633+bNsxHv7vdmwKn491L73+6MfDZC/d6fE+NiwZoaYOaF6JhcWbj/KvFfswV7oPvDd1eQal56BtRAhF5Vau7t+JV250/ZnRUi9Ll9qtoiVpqe8ztc63kiBCCCGEaPkCsUSGr0iAWWiyZ9hwjyUs6jbh89TkL6RrV6xHzmb2RV50EXG33EL+v/+NZedOcDhApyO8bx86Pvss4enpmudstzuYnfML/6nOMByV1o55twwmMjS41R98ifODloPsbUeKmPjaOuwKu/74qFC2aGgEtflQAXe+u5l+XWJ5c+qF59TP9fQ+83WA/LbhyTx1TZ9687t5/kYsVe6vYKgboH1/4yH++MnPbretURMwPlVSwZiXvqfE4j5b15vmWi2xwdzmwwUM6hZH28izJ0tOnrHw1rqDtScR2oQH85sBnXnk8l4tMjgrnx9CNK/GnGwUQgghhGhOgRhgNmYYegM2g8m4u/r3ccBUYBfwvMFk9FF1XS0AACAASURBVHyJMVKDWWjU9rqJ5C94U3F9Xe3vu4+ydeupPHBuMCi0Z0+SP/yAiv0HyHvxRcq3bMG8cSPmjRvrb+hwYNn5M8dnzqxtIlh16hSFH39Mu7vvRhekXOklKEjHn6/qTWxECC9/u5c1e08z+oXvqbTZKTRb/d4oy26xEBQe7vNxhaixaNNRxfUfbzpSG2Aut1YpBpcBCjzUrWy4fHD3eD67bwSxESG1wWUtGcmeylHc/f5mPph+ce2yDzYeVpz/uxsO1Qsw7zp+RjG4DPVrdV7UI56IED3lCqU66l563T4mjJsu6uZVfV5PNUI91TxujkvBa8oTZU/IOOf1Miaj/pUmP+zP5+Y3N2Kr8+I7Y6nivR8OsWrPKb64b2SLCwRJcFmI5uWuD4dcVSCEEEIIoclbwMvAbmOGoSvwGfA9cC/QBnhCzSBSg1lokjB9OmFpqS7XhaWlkjBtWr1l+thYkj/8gITp09DHxzuXxceTMH0ayR9+gD42lshBA+n+/kK6vPovgmJj3f7tir37yF+wANuZMxy+/XZOv/IvTjz1tKp563Q6HrosnUcv7wXAyZKK2uZlSnVYG8tWXEzeiy+yZ/gIdg8YyJ7hI8h78UVsxcU+GV+0Xg6Hg9V7TvHJVuWgcY3yyiqPjY5qAsYAaYkxRIYqB8YaE6zsEhdJTHVzwmKzld++tlZ1veOFGw4pjv3Dgfzany1Wm2KmMIDN4ahXxzk9KYZgvXKDz7r3OTUxhluHd1fcvmHA2N/1eT3VPA70BnMP/XdrveByXYcLzE1eQ1oI0Tq46sORPSFDgstCCCGEEOplADXN1K4HNhpMxiuBW4Ab1Q4iAWahiT42lu4LF7oMGNdkF7u6TeLMmaSvX0ev7dtIX7+OxJkz622r0+mIGTvWYzZy0ZKlBEVHEzFgAOj1RF0yqt56u4d6z2cq3AeQfdUoy1ZczKEpU8hf8Ca2ggLnsoIC8he8yaEpUyTILBS9ufYgt771I09//gslFs8nPOwOCFKOndYLnraLDuOWYdqCp1q9smIvB0+bXa7bc7KUa187W0bHYrVRplBbGJz3sSZgHB6iJyZc+eKb2IiQegHPi3smcMfIHoq38TZg7O/mWi29wZzFqpxBXrcJohBCNIZcVSCEEEII0Sh6oCZrbSzwZfXP+wHVTdAkwCw0UxMwdicoLMztOrvFoljfGZyBWofVSoe//IXkjz6izbhxmrKFPZUS+Ogn5UvvgXqZka7kz5/vskkhnM3CFsKda/p3Iiw4iNTEaPJLlTOTAaLCgpnQt6PiNk2dbbt06zHF9QdOl1FRdTZg3DZCOfjaMKP6pou6KW5/49Bz1zdFwNifmXT+DmD7k8Vqo6hc+WRJfp0seyGEEEIIIYQQTeZn4G5jhmEUzgBzTQOgzsBptYNIkz8RULQ2EazJFnYV0A1LS62XVa22UdaIlAQmX9iVK/p0qA1qaaknq/U+CNHQwdNlJCdEotM5U5Pr1rc9XlTOKyv28uer+hARevb1qbXRkVKDNm+ClWrfZ5v+OJZ2Mc7a5FqaFNbMvTGNnby5z4HW0C3Q5uPJoNnLPdaQVmqCKIQQQgghhBAtXYA2+bsE+BSIBd4xmIy3Vy+fA6QbTMbr1IwjAWYRUPJefFGxiWDsxGvp9NxzqrdPmD6NxJkza3/3FOSoKyYsmKv6d2J83w48m/OLqmCW3WJh94CBHsfutX2bYja3EK5OalwzoBNLthzlTHkV00b24E9X9a63faAET7UGE5sjQN7SArQtndaTCEIIIYQQQgjR2gRigBnAmGHQA20MJmNhnWXJgNlgMuapGUMCzCKgKGUkA+g7JNFzyRKCExIA7dnCnoIcl/dOosru4PvdebjpR3WOhoERyWAW3lIKuMZHhVJltzP7N335zYDOLm/f3MHTxgQTAylALnyvsVnnQgghhBBCCNFaBGqA2RckwCwCjq24mPwFCyhashRbQQH6+HiiRo3izLJlUFlJxODBdP/PWzjsds3ZwmqDHHlnLHyy9RiLNh9lX179bUNtVir11YEQh4MRJQd5LSuF2KwrATgy53lK3/mP2/lETbmVbn96Qu3DIc5DngK0tw7rzjO/6duEM9LG22CiBIxbJ3+VZRFCCCGEEEKIliBQAszGDMPnwBSDyXim+me3DCbjNWrGlACzCGj2ioqzweH/5XD8kUcAaDvpejo88wx7h49QzBYOiomh108/1lumJchRXlmF4c9fE11pZvLeFYw79BNtK8soCo1iefcLMetDmWr6hjOhUbwy/XlSuiXy67E8rv3PMySXnDxnPg7gx9se5/eP/967B0a0aq2hXq0EE4USOYkghBBCCCGEON8EUID5P8ADBpOxpPpntwwm421qxpQAs2hR8v72/+zdeXwTdfrA8U+SJul9AgUEWo5CI7pyilwiKF54cSkqHgh4766K7OLiua4Land1XReXBcQDLy5BERRUKP5AUVEQMKHlKGdpoUd6N2mS3x897JXMpE3aFJ736+VLkvnmm2+mk8nMM88831fI+d//AIifO5eKrFMeazCj0dDpb88TPbHxmuRqghwjnlzL3C9eaTRgfDy8PfHFORyM7sJLA28jM7wdAOG2Eianb+bKI9/XBKSPRHbku/jz+abf2IAPDorWo3aSPMvzV7eZAJ0EE4UQQgghhBBCnOsCJcDsD0GtPQAhvNH+kT9Snp5O0ebNZM2fT5dXX8GY1KvRms0agx6XzY4uJsZtf2qCXrNzv2s0uAzQpeg0lgGjsT86h6tPFbLvRAHfZ+RSZAhlad9xLO07Dr3Djl1XK2Oz2FYTcHMUFqKLiFD+4OKcEazXERtmUMxgbksB27Y0ViGEEEIIIYQQQnhHAsyiTdFotXR++SWO3Hor5ekHOPnkU3RbvIjCjRvr1GyOnjiB6ClTKNu3j4gxY5r1nsk/peL0sPz8w7voM7hbzeP65Q3qBJf5LThoXbuWrPkv0vW/bxBy0UU1y51lZWiDg5s1ZtG2TR7UxWMN5smDurbgaIQQQgghhBBCCHE2MiebjMCDwGigA6CtvdxkMV+sph+tchMhAosuPJwuCxagi4rCabVycs4TxN17L723byNpx3f03r6NDrNmYTjvPCKvvLLOa/M/XkPFmTM1j51lZY2+h9Nmo3TPXpxlZTjz3dd4BnDm5eEsL695PHlQF4/tJw/qirO4mKyXXsaRl8fpf/2Livx8slNSSBs2nP39+pM2bDjZKSk4rFal1SHOQg+O6kXv+PBGl/WOD+eBUT1beERCCCGEEEIIIYQ4Cy0CngJygC3AV/X+U0VqMIs2q/i7HRydPh0cDvRdK4O2NRnME8YTN3MmuqiomvYF69dz4rFZBHXsSPioSync9GWj7Ut+/JETjz6Gs6yMXpu/5uDYKz1OJKiLjaX39m01j60ldiYv3E5aVlGj7Z++7nzuGdGd8vR0sua/SPzTT3Hi4YcbLfNhTOpFwrJldT6HODfIJHlCCCGEEEIIIcTZIxBrMJuTTfnAjSaLObU5/UiAWbRpZ5Ys4fTLKY0uqx+cLfhiIydmz0aj1eJqJHO5ur3LbufAmMtxVVTQ5d+vUfrzzx4nEoybOYMOs2bVea5+cDA2zECQTkN2QTkaDbw86SImDazMdM5OSfG6f3FukUnyhBBCCCGEEEKIti1AA8xpwHiTxbyvOf1IgFm0aVkvp5C7RH1w9sSf/kzBJ58otrd+9hkhF16IoVs3HFYrR6ZObXKGcXVwsKDMzh2Ld7D7uBWNBv5580WM79+FtKHDvMqQFkIIIYQQQgghhBBtS4AGmCcBdwB3myxmzzViPZAAs2jTvA3ONjWY67BayVm8uMFEgnEzZnhVvsJaauf2xd+x90QBWg38a8L59LrjWsXX9dm9C63RqPp9PJFJBIUQQgghhBBCCCFaVoAGmCOBVcBlwCnAXnu5yWLuoaafIJ+PTIgW4iwr8xgsBnDk5uIsL0drNHrdvjZdVBQdZs2iw6xZjS5XKypEz7LpQ7h10Q7MmQU88rGZtRFR6ArdT+ani41tdnDZYbWSs2gR+as/9linWgghhBBCCCGEEEKcM94BzgdeBbKAJmUiS4BZtFna4GB0MTEeg8ba8PCa4Kya9mqCuc0N9kaHGnhvxhBuW/QdllOFfNxpAJMKN7tvP3FCs96vsRIfjtxcchYvoSg1VSYRFEIIIYQQQgghhDg3jQXGmCzmHc3pROujwQjRKpSCr86iIk7++c848vNVtW9uMFet2DADy2YMIalDOB/1Gs2RyPhG22njO+JyOGlOKZucRYsarR8NUJ5+gJzFi5vctxBCCCGEEEIIIYRos44C5c3tRALMok2LmzkTY1KvRpdpquoMW9d+wsHrr6fw6689tjcm9SJuxgy/jbW+duFG3ps5hLhO7Xh8xEMsTxpNvqGyFE++IYyvuwzAnp1F7ptvkvfe+01+n/xVq5u1XAghhBBCCCGEEEKclR4FXjInmxoPlqkkk/yJNs/dBHyx06ZRsG4d2f98BVdZGQCR119PzF13kvXX5ynbswdcLtBoCL7wQs77RwqGrl1bfPxPrdnLu98dqXmsd9ix6/QEOSt44vt3ucBVwMBV7xPUrp3XfTvLytjfr79iO19OIiiEEEIIIYQQQggh6grQSf4KASOgozKTuaL2cpPFHKmmHwkwi7NKYxPw2Y4c4eTcuZT+uLPyCZ0OHI4GrzUm9WqVesQDnt9EbrGt0WVBzgo6651sfWG829eX2R0E63Vul6cNHaZYd7r39m3qByyEEEIIIYQQQgghvBKgAea7PC03Wcxvq+lHAszinOByOsl7732yXnwRKirctoubOYMOs2a12LjK7A6Sn/pcsd2dQxOYMKALF8YZKNy4Ec2V41iQeoCVPx4np9hGXJiBSYO68OCoXkQaNOS+uwxtaCgxt9xMdkoKOYuXuO3b2Lcv3VcsR6P1TcUcZ1kZ2qryJEIIIYQQQgghhBAiMAPMviIBZnFO2T/kEpxWq9vlrZHN6ymDuTZjhY35O98mOXM/6/qP4z8JowEwOOzYdHoAeseHs8T5M0WLFqINDaXHuk/RhoVxZOpUtxP9RU2aRKfn/4pGo2nyZ3BYreQsWkT+6o9/K1MyYTxxM2e2eEa4EEIIIYQQQgghRKA5mwPMQa09ACFairOszGNwGcCRm0uF1UqQh6CorzN0Jw/qwsLUQ26XD0yIIaeonBPZFVidlVnGZ8oquGffOsYe+YFoWzH5hjA2JQxmuW0MK4YO47qY5YSPHo02NBRdVBQJy5Y1qFMdNf4mguLjiZ06tdnB5foBbEduLjmLl1CUmtoqZUeEEEIIIYQQQgghRMuQDGZxTlGqRwygjYgg8uqriRo/npD+/dBoNM3K0FUKSFtL7ExeuJ20rKIGy3rHh7PivmFEhgSx54SVdT8e4cCKNUxO20xiYVaD9hkR8Tx/5SNsfnQkxnZxdd6juqRGQUExkZFhNSU1okL1Ne3K9qehP+88dOHqL6gpleBo6bIjQgghhBBCCCGEEIHmbM5glgCzOKcoBUPrMyQkEHH1VRR+sRFbRkaD5e4mBvQ2IG0tsfNG6kFW/Hispqby5EFdeWBUz7oBYLuDlyY8zOT0LW7HvDxpNB/1u56BibFcnBiLqXMkL26wkJ7tPoAdFaqnPD2dI1PvQJ+QQLf/LUQXHQ1ASWExoRHu938yiaAQQgghhBBCCCGEZxJgPotIgPnc1lg5h2rGpF50fuVVilO3YF2zxm3N4vpiZ0wn/vHHVb+HUsmIMruDYL3O7fIdFw0istz9NpxvCOPWa59TNXaA+0f1ZM41yeSvWk3m3LnooqKI/tfrfPvOKuK3fUlUeRFWYzhZw6/g0qcfJ6ZjHM7ycsr27CH4ggvY36+/4nv02b0LrdGoekyeyCSCQgghhBBCCCGEaGskwHwWkQCzcFitDeoRR0+cQNyMGTWBX5fLRdnefVjXrCHv/fdB4XuiCQ1FFxaGNiwMR2Ehjpwct22bUzLCWVamKqC79811fH+iiB8ycsnIKfHYNkSv45Vb+nHBeZGE/99XlIdHs+8vz9Ip72SDtpkxnek771nyZz2Ks7SUpK2pHLr+Br9nMMskgkIIIYQQQgghhGjLAiXAbE42vam2rclivkdNOwkwi3Oas7zcY2at2oCuN5obcN1/yTCc+e4DutqYGPp8ux2ozIZOfupz1X3HhOq585d1XL13k9s2By67gaTvv8RVWsp5r75K2Z5fPJYdCWrfnoR338GQmKh6HLU1NyNcCCGEEEIIIYQQorUFUID503pPXQo4gT1Vjy8AtMBWk8V8g5o+tb4bnhBtj1LZBm1wMLqYGM9tIiI479VX6fTCC3SY/bjHtgCO3Fyc5eVejbO2mEkTFJZPrPl3sF5HbJjBY3udVoNepwEgr8TO0LTtHtu3//Zrurz6Cr22bCbyqiuJmzkTY1KvRttq9HoqTp/m2IMP4XI4PPbrTs6iRW7LlZSnHyBn8eIm9SuEEEIIIYQQQghxrjFZzNdX/wdsB74Aupgs5ktNFvOlQFfgc2CH2j4lwCyEguiJCgHdKbcQefVVlWU2pk9XFZBuTj1iTwFdY1Iv4mbMqPPc5EFdPPY3c2QP9j53FZ8+PIK/Xt2LKJvnDP+o8iKOdktGHx8PgC4qiphFb5E2ZjxWYzgAVmM4aWPGE/1iCkHt29Ppb8+j0bmvK+1J/qrVzVouhBBCCCGEEEIIIRr1B+BZk8VcEwyq+vfzwO/VdhLkh4EJcVaJmzmTotRUtyUa6gd0oydO8FgywllYSPYrr9L+9w+jCfL+K6iLiiJh2TLFOtLVHhzVi82WbNKyihr01Ts+nAdG9cQYpOPCLlFc2CWKHcYw5UkE//sDpk6R3NivM6P6tOePH+wlLXI4XDMcvcOOXaev7H+/keVrPyM0NqLm9S67HajMbgb3k/a5nE4KN2/2WN8ZfssI99UkgkIIIYQQQgghhBDniHCgM/Brvec7AaFqO5EazEKooGZiwNpt3dUM1hiNuKrKY4QOHkznlBT08R2aNTY1wVVriZ03Ug+y4sdj5BTbiAszMHlQVx4Y1ZOoUH2dtmsffILeX69x29fHyWP4X/K1qsd3/6iezLkmuebxqb8+T9n+/QSbkilYv8HtpH0nHp9Nwbp1oNdDVVC6Mb6YRFAIIYQQQgghhBDCnwKlBnNt5mTTW8DlwGzgu6qnLwFeBDabLOa71fQjAWYhvKQmoOsuIB1z113kLHiDvPffB0AXF8d5KS8TNnTob/27yej1lTK7g2C9+3IVeady+Hn8zXTKO9lgWWZMZy5a/RHppRrW7jrJ+j2ZWEvdB38B4sIM7HxqLAClu3aRMeVWt21rT9pXsH49Jx6bRVB8PBVZWe77nzmDDrNmeRyDEEIIIYQQQgghRGsK0ABzCPAP4B6gOgOxAlgCPG6ymEvU9CMBZiH8rLGAdMGGDWQ++RTO4mLQaIidPh2cTqxr1rjN6G1Jeady+Oav/6DDtk1ElRdhNYaTPXwsI5+eRUzHuN8+R6md3z23UbE/y/NX1wS1j957H8Vbt7ptWx0wdtntlKenoz/vPLcZ4cakXnSaN4+CTz+lw+zZTSo5IoQQQgghhBBCCOFvgRhgrmZONoUBPaseHqxdk1kNCTAL0UpsGRkcf+RRyi0Wt21qZ/S64++M55KiEkLD3ZfdGfD8JnKLbW6XB+u17PjLFUSFVF4ISxs6zGNd5cZKXrjLCI+aNImjd0+jIjOTqAkT6Pz3F7z8dEIIIYQQQgghhBD+F+AB5nZUBph3mSzmcm9fLwFmIVqRs6yMjFumUL5/v9s2jZWAcFit5CxaRP7qj1s943neBjMLUw95bBMdquf3Y5K4rV8HMgYNUuyzz+5dbsuQ1M4Id7lcZL/4EnkffUS3/y0kdPBg7z+AEEIIIYQQQgghhJ/5IsBsTjY9Afwd+I/JYn646jkN8AxwLxAD7AAeMlnM+1T0FwG8CUwEXECSyWI+ZE42/Rc4ZbKYn1UzLm0TPosQwke0wcFUZGd7bJP77jIK1q/HdvQoLperZhLBnMVLcOTmAuDIzSVn8RKOTJ2Kw2ptiaHXeHBUL3rHhze6LDbMgDFIQ36JnefX/cqVC3bgiPAcANfFxnqscV17mUajocOf/0SPNR/7NLjsLCvzWV9CCCGEEEIIIYQQzWVONl1CZRD5l3qL/gTMAn4PDAaygU1VwWMlLwKdgQFAaa3n1wHj1Y5NAsxCtCJnWZnHchEArrIyTjw2i4NXXkXaJUM5PH5Co/WIAcrTD5CzeLE/hupWVKieFfcN4/5RPYkLMwCVE/vdP6onm2ddxtY/jeGWQV3RauBYbimrO/b32F/IDTd59f4ajQZDQkKd5/LXrKkJEqsNFjusVrJTUkgbNpz9/fqTNmw42SkpLR6wF0IIIYQQQgghhKjNnGyKAt6jcjK+vFrPa4BHgPkmi3mVyWLeC9wFRAC3qej6BuARk8W8i8oM5pqugR5qxyczYgnRirTBwehiYjwHmYOC0Gi1uGw2nFYrToWAZ/6q1Q1KatTmj5rNUaF65lyTzJxrkimzO2om9AOIQs+Lk37HtBGJzN9gYbltDINPmUkszGrQT0ZEPL8mjcb96JWdXrCAM6/9m9P/eg1XVQBfqYRIdVZ47cB9dVZ4UWqqYh1sIYQQQgghhBBCCAUajUbTu9bjHJfLlaPytf8DVpos5s3mZNMztZ7vDnQENlY/YbKYS83Jpq3AMGChQr8xQGNjiAAcKscmGcxCtLboiRM8Lo+bdjd9dv5I949XE//0U4r9OXJzKd75E7Xrq7dkdm7t4HJtyR0jeWvaxWgiI5k98iGWJ40m31BZeijfEMbypNHMHvkQ7//qOaO7zK6wf6v63BWZmTWBe08lRFx2O2cWLAiorHAhhBBCCCGEEEKcdYKA/bX++72aF5mTTTOBXsCTjSzuWPX/+ll8WbWWefIDlVnM1aqDSfcB29WMDySDWYhWFzdzJkWpqY0GOI1JvYibMQONXk+wyUSwycSZf7+uWFbj6O23o+/ShcirryJ0+AiyXngB24HWz84tszsoLKsAQyhL+45jad9x6B127Dp9TZuiYhvr92RyhSkeQ1DlNTBriZ0FqQdY+eNxcoptxIUZmDSoCw+O6kVUqL7Oe7hKS3GnOlhcneGd+cyz5H/0ERqD3u1rQDkrXAghhBBCCCGEEEJBBXBBrceK2cvmZFMfKif1G2GymO1+GNNfgC/Myaa+VMaJH6v698XApWo7kQxmIVqZLiqKhGXLiJs5A11sbOVzsbHEzZzRaPBXKeNZG1FZw91+/Dg5i5dwbNq0OsHl2tRk53o74Z2n9sF6HbFVdZqr1Q4uV3vwvZ+4+O9fMvfjPWzZn83khdtZmHqInGIbADnFNhamHmLywu1YS+ruX/NWrvY4vryVq2r+XV0qxGXzvI925ObiLC93u1wmBRRCCCGEEEIIIYQCl8vlSqv1n5ryGEOBdsA+c7KpwpxsqgBGAQ9W/bu6j/h6r4sHTil1brKYt1NZSsMAHAQuB04CQ00W80+qPhWgqX0b/bkgLCzMVVxc3NrDEMItZ3k5WqPR7fLG6gVXMyb1otu772I/foLCLz6n4PMvsB875vH9dDEx9P627l0PDquVnEWLyF/9MY7cXFU1jNW2n7fBzMLUQ27H0ykymMwC9QHb+0f1ZM41yUBloHd/P8+TCAL02b0LrdFI+YED2LOyOPHoYzgLCty218XG0nv7NnLffhtj796EDhmCs7DQq3VUmz/qYAshhBBCCCGEECJwaTSaEpfLFebNa8zJpmigS72nlwLpVGY276MyIPxvk8X896rXBAPZwGyTxaxUg9knJMAsRBvksFrJWbyY/FWrfwtuTpxA3IwZdYKbjtJS0voPUOwvqFtXQi/8HSG/uxB99x5kv/gitoMHG7QzJvVqkFWtFPCu395aYmfywu2kZRU1aN87PpwV9w2jsNzO2l0nWfPzCdKzG7arM3athgHdYkADGuDR//yeKJv777jVGM4lu3+o81x2Sgo5i5e4fU3czBnE3n036aMug4oKOj73HHnL3lX9mcH7oL0QQgghhBBCCCHOHk0JMDfGnGzaAuw1WcwPVz3+M5WlLqYBaVTWar4U6GOymAsV+poM2EwW89p6z98I6E0W80o1Y5IAsxBtnFLGc9rQYYo1m70RMngw4SNH1kymV7R1K6U7d7ptHzdzRoP6xdYSO2+kHmTFj8dqaipPHtSVB0b1rFNTudRWgenpL7wa3z371jE5fYvb5cuTRjMq5VmG92qHXldZJchhtXLottupaCSoHtSzJz3efw97ZiaZTz6F7fBhoidNJPftd1R/Zm+D8EIIIYQQQgghhDi7+DHArAGeoXJivhhgB/CQyWLeq6KvfcBjJov5i3rPXwG8arKYL2j8lXVJgFmIs5xSdm7ktddiTE6mbM8eSvfsoeKUYoker1SXl3CnzO4gWK9zu3zA85vIraq93JgQvY57RiTiclVOdbp88z7+9vW/SSysP4EqZETEM3vkQxQZQokMDuKyPh243NSBAd1iePh/WxiwbR1XHvmeaFsx+YYwNiZczE/Dr+OdP4ytCXzbs7I5fNNNHoP2uthYzvvnP8h9+x0M3buj0WoUM6RlEkEhhBBCCCGEEOLs5asAsy+Zk02lgMlkMWfUez4R+NVkMYeq6SfI90MTQgSSuJkzKUpNdZs92/GZp2uyZ9XWMDYk90Gj1eJyOrFZ9nts68jNJffdd4kaPx5deHjDvhx28BBgnjyoi8eazXcNS2T2Vck1j50uF7PtDzE5fXODYPGKpNEYoqOhxEZBWQWf7D7JJ7tPoqEyOL277ziW9h2H3mH/bfLBfBdvpB6sqfOsi4pUzAh35OZSumcvRZs3Yzh8GIfV6rF9/qrVbSrALDWkhRBCCCGEEEKIs0IekARk1Hu+N+CxvEZtksEsxDlAbc1mUC6pyz/oiQAAIABJREFUUT8jWW0JDk1oKFHjriX6likYunZRXY9YTc3m2mU16revHSyubl9QZucrcxZfmrPZcTgHu8PzfjAuzMDOp8aq/sy62Fg6v/wShV9sRBsWRu7SpYrrp3riwcZ4G9D1RwC4OTWkA2H8QgghhBBCCCFEawrQDOY3gJHABJPFnFb1XB9gFbDNZDHfp6YfCTALcY5RqtmsZsK72tm2Su0NvXphP3oUl+23MhcaoxFXeXmDtu7qEaut2dyU9qcLyxj8wldux19ty+xRJMaFq/rM9deRmoB07J13YD91iujJkwnp29frgK4/JxFsSg3pQBq/kCC/EEIIIYQQQrS2AA0wRwIbgCFAZtXTnYDvgatNFnOBmn4kwCyEqMPbYKKa9i6nE+uateR/9BG2jAyP769Uj7iksJjQCPX7YzXtleo8VxucGMP1F3VmWAcDR6feQae8kw3aZMZ0pv/Hy4npGFfznFJAOnb6PVg/+QTH6TPE3X8fcdOm+fxv4ClIqxRM9Dag7o9tyNdB5nMhgCpBfiFEIDkX9rtCCCGEEJ4EYoC5mjnZNBboV/XwZ+Ark8WsOmgsAWYhRAPelNTwpr3L5SLt4iE4C92X8dGGhdFj/Xr08R3q9u/HQNm8DWaPdZ4NOg22emU0wm0lbus8Tx17YU3N5urxeAqgdl2yBOvatVg/XkOX/7yOddUqjwHdmDvvJLhPHzQGAxFjRnPmjTe8nkTQm3WklIGtCQ0l6rrrCBt6CZHXXOPzLHhfTYJ4LgVQ20KQX7QuCfaJ5lKzDZ1L+10hhAhE8nsvRGAJ5ABzc0mAWQjhkVJJDW/aq51EECD4/PMJv2wUIYMGkfX3edgO+C9QplTnedmMIfxyzMqnv5xk069ZlNgcddrUmRSQhjWbq8elFIR3uVxoNBrFgK42Kgpn1cSBvTZ/zeEJExVLcCQsexdDQgIanU5xHXWaP5/Sn3cRFBtD+Jgxqv9m0bdOodMzzygHpI1Ggtq1I/KG6+nwxz96Xfe7Kc61AKqaoH37P/yBnKVvoQkKwnbsGPkffOCxvUxE2fZJsC+wtMXt1Jtt6Fzb7wohRKCQ33shAlegBpjNyaYhwOVAB0Bbe5nJYv6Dmj4kwCyEaFFqJwVUy1fZsLXrNhcWFBMRGdZo3ea8Yhv9n9+kOK7XpvTjsuQORAbra/pfkHqAlT8ep6CgmMjIMCYN6sKDo3rV6V9tEF7Xrh3Y7SR+vJqDYy5XbA+gCQmhx8eryV+50uM6MpqSKTdbCOnfn8QP3lcOGIeEEHbJJURccQWR465VH5Cecgvxc+aoau9pEkQ1WipLuqV4Ckw5S0o4cPkVikH7nhs3kjZoEADaiAiPdxb4Isjvb3Iy5ZkE+wIjoNuWt1Nvt6Gzbb97LgqE74xo22QbUubrdXS2/d7LNqRM1lHbEogBZnOy6XHgJeAAcBKoHSh2mSzmMWr6kQCzEKJFKdYjnjaNsGHDKNqyhaLUVOzHjyv2qQkNRaPT1WTn4mG/5i5Q5s1Jv9qazTqthoHdYhjSI5a1u05yNLekQZve8eGsuG9YnSCztxm9ihnP4eE4i4rQ6PX0+Wkn6ZeOUmyPTkf48OF0TnmZ0//8p08nNdRGRNBh1mMYe/YkdPBg5YsOOh3hl11G+z/8geA+vRssVjqoKtu/n4zJN9eZaLLBWygEUANhkjylbdRls5H53HMUrN+Aq7RUsb9e2/6PYzNm4rKVYzvovkRMteYG+ZtDaX2ebSdT/nCuBvsCKaDb1rdTNdtQ9KRJFH+3A1xOTv/rNb/fndLS/H0SHwhBgkD6zgj/a43jFeHfdXQ2/N43d/0Ewr7U386m79m58PeqLUADzMeAF00W8+vN6UcCzEKIFuXNCbajtJS0/gN8PobQEcMJGzSIkP4DCPndhbhsNq9O+pVqNneKDCa7qByHU93+9fYh3Zg7zkSoIQiAY/NeoujtpW7bh999D13nzK55rDyJ4HSiJ4zHduwYYZdcoipjuPdPO9GFhgL+z1pTal8t4f33CR3Qv2ZM7g6qtJGRaDSamtcVbNjAiUcfU+y/fgC1JSfJ81UA9fDkmynbsweNweBVQF3NRY2Et9+iZOdOosaPbxBobu0T1LPhZMrfWqIUTaAJtIBuW99O1WxD8U88wcnZs9FERuIqUJ5wvDUvXIH/60i3tTrVgfadEf7hz22uLWxDrR3M8vfk3G39976p6yeQ9qX1SZZ6Q4H89/K3AA0wW4H+JotZOevIAwkwCyFanDeTCKrJzu30wgvgqMDlcHDq2edwevMdDwpCFxOD4/Rpt03qn/Qr1Wxecd8w0MD2A2fYsv80y3ce85RUXSNYryUuzIiuuJAnPn+FxMKsBm0yIuL5dU4KsyYOqnnO24OMphx4evM38+UEc4aePYm9Yyqle/bQ+YUXatpn3DIFW0ZGg/baqCg0wcH0+Hg1QbGxQGVd8P0DB0FFharPXJSaCsHBZD3/N7/W/vZHALXgi43YT57EfuwYee+/r9jem/7tJ05SsH49IYMGkrhsWcCcoJb+8gtH7p6Gq6ThHQLV1JxM+TtLvTVPaNWW3vFlsC8Q1megBXQD/aTf3Tp12e0U//ADx+6ZrthHl/++wann/oouOpqKU6cUP2+3JYvB5SL4/PNVj8fb8dfn7zrSbblOdaB9Z4Tv+Xuba+425K/fSn9fKPJGU9aRmvEXbNhA3ocfUbJjh+IYWvvinidNXT+BtC+tHlOgZqm39t04gfj3akkBGmD+L/CLyWJe0Jx+JMAshGhVSpMI+jobNmzECILi4ij5+WfsR4+qGmNjJ/1qazaX2R0kP/W5qvepLdxWwuT0zVx55HuibcXkG8LYmHAxK5JGo4+O4uenr6zT3psAsLcZ0vWpmfjRm/F4215NxnPHZ58hZsoU1a+p3o5cDgcHrhhLRWamx/5r2tvt2LOyOPPfhVhXrlRsX/vzqjmwcpaUUGaxcGzmvR4vnNTfRn0Z5Dcm9aLbO++Q/eJLWD/9lI7PPkPklVcGzAlq5tPPkL98uWKfjZ1M+TtLPZBOaNXUUu/zw/dogoKa/B6Bsj6dxcVUnDlDxpRbAyagG6hBfqV1WpaWxtG7p+HIza0zwWxj6q9PNd/jMst+ir/5hpipU+n45NwW2YZ8eUdO1ORJxD/+eM2dM2r614aGotHrVa+jlgwSBPpFENF8/r6I0OQkBj/fJeDPC0Xe8nYdqR3/yT//GevaT0CnA4ejQdva/Sdt2Uzxt98Sdumlde76g9bP8FYzWXjXNxYQNmxYzXPHH3uMwvUb3L7G1xfH/B1A9UeWeiDdjXOuX8wM0ADzXOARYCPwC2CvvdxkMf9TTT8SYBZCBDRfB8pqt684fZriHTs4+bj7YGq1HhvWY+zevc77+Kpmc3SInoV3DCS32MapgjKe+/TXOsv1Djt2nb7Oc8N7xjHud525sm887cKNqicRBEhZ9QN9589WnSHdXGoC0t60VxMo67F2DYZu3WqeU7td2E+c4Mi0exQvPlQfuJUfPMihcdcpfqbq9i6Hg6KtWync9CXW1avdtq8+sLKu+4yTjz+u2D+4KfHh4yC/7dgxguLjOfPaawFzgpqzeDHZ/3wFnE5V7Yu3b0cXG0tQx44cveMOv+xbmtK++jW+OqF1uVxYV63Cduw4HR59RNWFmc4vzifqxhu9ep/aY/fn+lHbPufNpWSnpGA0mSjft09x3J4Cur4+yd4/cJDHC0XaqCj67PiuWe/h6+xZbUgI6aMuw5GXR9jIkRR/843b9/b2QlrXRYs4evc0bBkZnPfqK4QNHer375jS98CQmIiuXRwJ776LRqNRPTFx0vZtBMXGKn/PjEZwOknevQuNVqv8exYaSvePPsTQq1dNEMhfQYLWuAhyNmjtTEBv2/vzIkJFfj7plwxVbNdn9y7K9u5FFxODLjbWq99i8P474G0wy5/Zld58zzR6PZlPPoXtyBFKd+5UHH/xt9+S++4yAIq+/tpje323bpx66mmM55tI/PBDXKWlAVGuQO366fDnPxM37e6ax+a+FygG1Zs714q/A6hq+ne5XGTNm0/eO+94HCv8tq92ORw4i4oC4m4cl9OJ/WQmh8ePb/OTizdHgAaYD3tY7DJZzD3U9CMBZiFEwPNnNiwoH2xXC/7d74i8+mrChg3l5OzZPqvZfP+onsy5JrnmsdpJBAG0GhjQLZqjuaVkF5Y3WN47PpwPZl5ChdNFdkE5p4vK+P37P6MpKnSbIW2MiWbnU2PdvmeZ3UGwXqdqfL7WnBNgtduF2trffXbvoiIri4NXXqVq7H1278KRm8uB0cqT8DYIYGs0TZq8slpLB/mbc2BoP5PDgREjFNvV/hurzlB3uTh0zbXYMjIIGTiA0p0/Kb6mmjfv4Sot5fTrr5P7pvs7BXx9Qlv/5Cj/4zVkPvEEaDQkvLcMY8+ebvvXRkZiTEoi4d130Gi1qvqvT+36sZ86Rcn331Pw+ReKJ8AdZs2i4swZSn74AevaTyjaskWxfcHGjZz4wx9Br0cXHq64ncZMuQWXvYLoyZMwdOvm16y1rJdTyF3ifh2FXnwxCe+83eT+vdmGXBUVZL30sseT1Op1WrR1K4bu3dFFRjbtBNXDftfldFK8bTthlwzh9L/+5TlbeMIENDotaLR0+utzittc5I03YktPx2krp8eaNWiCglT/3vf+7ls0wcGqfm/Qakneu0dVwLha0rfb0YaEqO6/9/ffowsPqywRdftU1eWbwH2QIGbqVIq/+46IMWPQRUYCKo6H9HpMe35RHvNZzt+1Q/12d4cPLiK4+y0o2raNk3Pm4DiTo+p45dANN1KelkbIgAGU/qT+t1jNfg6Xi8KvN1Oenk6HWY+RPvJSz78FMTGEjxmNoVsC0TdPJnfJEr9dRHcUFJA+YqSqeTLsmZleHTPWvIeKdZT14ktYV68m8tpr6PjMM37NtvWW4oU3vZ4ub7xB+IjhNe/flO3aX+WMbMePc/jGm3x256Ghe3cSP/ygpv+MqVMp/dH9BYfa/bscDtIvuwxNkN7j3ZnNPSZVXcpv/XpOPKbuu3M2X8wMxACzr0iAWQjRpvg6UAYqSi4YjVDeMHjrTlNqNtfOMq4fkDY47NhqZTBfYeqAIUjL15ZsyuzuszbVaixD+svHLqVXh4g6n6E6Qzqn2EZcmMFthrS/+SK46asAqstmw3biBBlTblV163jpnj1k3DLFY7ZttT67d6EJCsKRl0fuW28FzK1kag/ke27ZjKFjR6/7L927j4xJkzy2aertoxW5uRyeOImKzEy0ERGK2RM9P9/Aidmz0Wi0lPz8s+LfuMdn60gfNhxcLrSRkTg9THJW/Rkyn3sOXXgEFXm5WFeuctve22wXbUgIhydOwpCYSPyTT6KP7+Ax2KcJCUFrMNT0XfzddwR16oR1xQrFbBrbwYNk3DJF1clU4ddfc/zBh9y2q9+++NtvOTrtHtXtHfn5lPz0MyH9LiL3zTc9T4B6993kr1yJs6iI9rMeI+bmm312ku2y2yszyTSamiwrT8FBTUgIvTZtJKhdu8rXVx2f17512VdB/sxnniV/+XI0er1Xk4FWfwZvLuDW1tz9bs13VqfDtG+vcvta38E+O38EnU7VvivugfuJu+sudNHRyu8RHU23RYsIufAC1fvGLv99g7ChQ9EajcoBXY2GkIsuIvHDDwAVJTsmTaLDrMcIiokBPO8b0WrB6aTjc88Rc8vNqvoPv3wMXf/zH7fLW/vW+pbg79qhvry7Q5+QQPT48cTdd2/NvmT/xUM8/jZpDAaSf9ndYExKgTjbkSMcvHZc5fGNh/hC3MwZxN13HwfHXokjLw9teDjOoobHx9V0sbEkfbOVMwveQBNsxH7iJPkffuix/6jx4zl07TgAEj9eTcb4CW7b19crNZXDN93kt4vomc88S/5HH3lsU72vth07RtaLL1H05ZeK/TblLraSn35GFxONddUqv2TbutPYfqLizJma37+mZP+q+T3o8/1vtanzPvqI3Lfexna4YdJmoxdk/z5P9dwmeR98wKnn/uq2bbXqv1npvn2cnPMEtvR0Vf1bP/mE3Pfep2z3bsX2ao6p4bdt2llaSunu3VjXrsX68RrF/u0nTpC3fAW5S5eqOp4o25/GYRV3ykkGc9slAWYhxDlPsf7s0rcoM/9KweefU/jlVx6DTNC8ms3Vbe98bRMDt33K2CM/1GQYb0oYzM7h1/POH8YSFaqnxFZB6v7T/PHDXdgc6gLNWg248HjsX6NPfASXmzowpEccf1v3K+nZ6gLk9fk647kl6nb5uvZ37fauigrSho/wqpZpoE2GoSa7RKPX0/7RR4m9Y2qD5fVPLlwuV51A2qEJEyn/9dcGr6vm9mRHReDL5XJRsvMnjk5tOK76emzayKGxVyq2q5b0/Q7SLx7SpPZqTrKrA65B7dujjYri2D33eM7i0mhqMhPr8xTssx0/zqEbbsRVXt7oLae1t7mCTZs48fs/qPm49Nm9i+Lt2zn5xF9w5ueral+2dy9H770Pl4pjt8ZOsD19b7osWED+8uVY131G9+Ufkfv22z47yUanI2fh/9AYjfRY9ymGrl1/a69iO81fswbrmrW0f+SPFH35pceT+IozZyjZ+ROZc+eq2oay5r9I7ltvKa7PxtZpbd5e8PVEbXA2/PLL0Rj0dJ43T1X7dg8/jDYsjJjbblUV0G1KHena24Sv+4+dMZ24adMIiotT1T9AyMCBJL63TFX/aDTE3XsvHR59BFAOViZ+9CFB0dEAOEtKyJo3j+hbbqHw889b5Nb6QAhg+/sYxNfHHwCJH31IyEUXqWofOW4c5/0jpebxqRdfpHD9BiqyGpZV03frSvcVK2r+zvmrVmM830Tmn/6kfKu8w0HJzz9zdOodHscO0GvbNg4Mr8xWVXNxOGlrKumjR2NISCD+iSc4NmOm5+BjVCShgwdTkXmKbsveVX0XW1P2fxV5eRyeOAmXzYbjzJkGy301OXdtvkiqCL/sMgyJiUTdeCPaYKPPyi1E33ILZxYupOCTT+m+dg3G7t39Uv4odsZ04qtKzpUfOlRzAcKd2t+z/FWryJz7pMf2dSYL/79tHJs5U/Wdh3krVnDqqadV9w/qzwscBQUUfvV15R1tCvrs3oUtI4PDN96k2LYmcUZlALu6f41WS/mBA1jXfuLxOERqMLcOc7KpNzAJ6AYYai8zWczK2R5A4/dBCiHEOUQXFUXCsmXEzZyBLja28rnYWOJmziBh2TKC2sURPnIknV94gV6bNir258jNJe+DD7CfOlXzXLi9hHv2reODDc+y9tMn+GDDs9yzbx3h9pIGrw+3l5DyzQImp28h2lYZVIm2FTM5fQsp3yyoeU2oIYjRyR1UBZfXPDicH5+8gvQXruXeSz2XUDLoqrJcsgpZsOUgd735faPBZYC0rCLeSD3Y4HlriZ15G8wMfH4TyU99zsDnNzFvgxlrib2RXrwTN3MmxqRejS4zJvUibsaMFn8Pb9prgoKImez5YCx6Yt1sG6VttKVnWq4/vvpcVAYftOHhNc85rFayU1JIGzac/f36kzZsOKf+9gInn3yqwYF7wtI3vf4b66Ki6DBrFr23b6PP7l303r6NDrNmNVg3Go2GsEED0VVl+Lmji40lKDqamKlTK4NTCutYFxuLLiKCrgv/S8Kyd9FVBWE8tde4XERcdRXBA/p7DAxC5X7FUVTE0ekzOHTd9Zx6/m+NZyUC5ekHyFm82G1wGfB4slmell6ZieKmnmF1/wBhF19cOaFQvUmC6tPFxqI1GokYPZo+332rav1rjUZCBw4keeePqtvXeU7he2Po2pUOs2bR66svCWrfnvxV7uuiA+SvWEnO0rco3r4dV0VFzQlezuIlOHJzgcq/U87iJRRu+hJdXBxRN1xf53ugZjt1FheTnfIPSr77jmPTZzTa/5GpU3FUXaQ6/dq/OfHHP6rahpzl5URNGE/XRYvQetg+3K3T2nx566o2OFjV37jrf16nyyuvqG7f/uGHiJt2d81YlfZd9Zd7+1vg6/7bzZxZE1x2lpWpKr9Re3tT2qZ1UVE1weXqx+6+M92Xf1QTXAbIeest8lesJGPyzYrbqDvOsjLFz9PYb0d2Sopi302lNCbF/YTC8ub2n7N4CbnL3lPdHo0Ge+Zvx6NK21zHp5+qeWw/eZK8pW81GlwGsB89VvNbAJXbd4jJpOp4RaPTETZokLp9uz6IkIsuwpCU5DG4DJXbn8vhIGnrVhKXLSOkb1/F72XMzTfT9fXX6b5qJbqQEMUxaaOisB875nZ57b+x7cgRyg/9dldiUEwMPT/fQM/P1qk+pvN2v9JgvB721Wr2K47cXKyrV3P6n//EYc0nZ9EixeOPOq/38Ft57P4HKNr6DS6bjdy3K0tENeWYV82+tGaMjdxFVF/t75U2TDkOWP37ChA2fBhx0z3H4mr/zZS2t/r9g/p1pIuMJHr8TaqPodTuV6vHE9S+PSEDBoDe892s1f1r9HqCTSbaPXC/38/lhHfMyaZxVE7udz1wD9AHuBYYD7RT249kMAshRD3NvdJfm7F3b0IuHkzx5i3YT5xouLwZdayqKdVsjgsz1KmprFSy4/2Zl5CWVchX5my+MmeRkdMwCF5bZHAQ3/xpTE0Ws7clQZqiObdp++s9vGnf7Hq7PswcbAql8Z/3+usUfrGRuHumVZb5sFrJuO12bAcbXoyolvjhB4T061fnPfz5N/ZnlnpT2qvJIOq+cgUHrhgLTqfqEhxNtX/IEJxWdf2XHzhA/scfk7vkTbftW3p9NsbT90ZtBi1UZuj3+WmnYs3gmDvvpONflLOG6nO5XBR99RUn5z7p8U6H6s+cv2YNmU9WBYUqKty2b252rr+1xDbR5Mk3/bhv96Z/xf1EdDTdV69C37mzb+rtevjOWNd9RubTT+MqcX+M0Nxb61vq7h1PY3LZ7Vg/XUf5/v10fPYZVeu043PPEXXduJrAlK9rJLd/9BHa3Xdf82rPqtjmyg8e5NBN48HuPjmgudmz/r5LAHxfT9bQqxe2AweImTqVjk/OrXmP+n/jkH79KP72WwzdupG4YnmdMlS1Ka0jf38PFMtLREURdvFgyg8fpvvq1RyomvjVHV10NC6HA33HjnT97xvkffCB59I7Y8YQNuRiom+9tdF1pPaYV+127e33xlFQwIErxnp1zOXt38zfWerefM+cNhvpIy/16k7LJk9q6OdzuUAViBnM5mTTTmClyWKeZ042FQIXASeBd4FvTRbzP9X0IwFmIYTwktKPqL5bNypOn8ZVWqqqP33Xrhi6dcNlt+Oy2Sjds8ermZC9nUQQ6pbsqK6p3FjJjlJbBaanv1D1ObrGhnBB5yjOFJXzQ4b7g6TGxtMcLRFs9Uft77Z+YOXN+JW+MyGDBtFtyeIWuRW/9vh9VefSF+1VTyJot1N+9CiHx12n+BmbegtvU4IWgbY+m0LxJDssDF1MDNrICHqsXu3XyS7VjKemZmJVttyZ11/36gQv0ErvtNQ24c860v7uvyWCcd5Iu2QoDg/lbpobdPHJhSWFshpKY2r/yCMcf+hhAHpsWM+R2273nGSg04HDQdRNN9F5/jzPZUe6diVs1KUE9+xJzK23Aur2Q4kffoAxKUlV++YEpnxxkUJJS20TvrpQZOjZE2d5ORXHj9Px2WeImTLFc61zQBsTQ8LbbxHcu7fbcSvx5zGjV8FHLy7GAvT+4XsOXnmVX/dDjfH1ZNX+DqD6+4Kvv79nbT1xpqUFaIC5CPidyWI+ZE425QKXmizmveZk04XAZyaLuZuafiTALIQQXlLzI6oJCaF0506Ktn5D7jtvg8oayWr1/vEHdFW3wdbPGK49KaCajOGSwmJCI9z/xillSHurfkZ1fb6u2dwWtPUDK18fyLcUf2ape9u+pbNdlDQ1SyxQ1mdTqA7y22y4nE6/Bl5aIshf/ZpAutDV0tuEv/e9vu6/NQK07nizjWoMBo7ePQ2H1Uq5xaI4ntK9+zjzxhsUpaZ6lZVfzZssacXardOnU7R5MyH9+tHu3pnkr1zpOckgIQH7iRN0W/Q/woYOVVUj2ZiURI9PP1E1Hn/c3eFJS/x+t+QdYND8C0Xa0FAKNm4k4vLL0QYHK/4NYqbeTscnPdfw9UZr71fU3EkRP3cuFdnZxNx+m98vUjRFoAVQW+KCbyDdjXOuC9AAcyZwucli/tWcbNoHzDVZzGvMyab+wFaTxRyhph8JMAshRBP4+jasmKm3ow0JQaPXk7P0LcXsZ110NBHXXE3UddcR0r8/+dl5fPPXf9Bh2yaiyouwGsPJHj6WkU/PIqZjXOPjV3nyVT9DunYAG2DasETGnh/PvpMF7D6ez7pfMuu8vn57gNTZl5EQ99vvqrXEzoLUA6z88XhNRvWkQV14cFSvZpfTEK2rJTKgfMEfWeretg+kbJfm9h8I69Nb52KQv7ZAu9AVCNtEIGrpYJwnardR25EjHLzqasX+qtuX/PQzR267TdUY+uzeRUV2NkWpWwkfOQJddLTiZ7afOkXZ3n0EX3gBR++6W/EzJG37v5qJaNWsU1dFBbqYGDRarXJZNa2WuBkzaP/7h9Ho9QF3d0dLl9IJxDvA2upFdG/48/gjENdPIAZQW3K7bu27cc51ARpgXgOsN1nM/zMnm14CJgLvUFmDOdtkMaua9bzFA8yJcz57EJgNdAL2AY9kzB/3jYf2BuBJ4A6gM5AFpGTMH/daU95fAsxCCF9r6duw6gvq2BFXeXmj7+GLkxFriZ07X9vEwG2fMvbID0Tbisk3hLEpYTA7h1/PO38YWycIPOD5Tdjy8rk5/esG7ZcnjaHIEFrZrls0V1/QkeE92/Ho8l1+rdksWlcgnlwEutbOdgm08gkt5VwO8ou2p7WDcWq3UfupU+S+u4zcJcrHNn1278JZUkJ2SgoF6zd4vOBe/duR8+ZSsl96qXJieyMIAAAgAElEQVRizZtuVKwHX/jV19gOHaLdQw9y5j8LVI2pKTWM/V0juantvRHovwWtvZ9rKxfRveHr449Aq/dfLZADqK29XdcXaONp6wI0wNwDCDdZzL+Yk02hwD+A4UAa8JjJYj6qpp8WDTAnzvnsFmAZ8CDwf1X/nwacnzF/XKMDTpzz2WqgCzAXSAfigZCM+eO2NGUMEmAWQrQ0X9+G1emllylOTcW67lNsB9xPmlYtZPBgIsdegcZgQGMwUrhxI0Vbtng1nkO33U5FIxO0BfXsSY/336tzIJay6gf6zp9NYmHDWcczIuJ5YtTD5AeFKI67mlLN5nOxpEZbE6gnF23d2ZRNE4gkyC/ONgF/a30T656e+e9/yXlzKRGjL6No6zeK7xE2fDhFW7YQe8dU8j740K+Ta/l78q7mtlfjXP8tUHIuXkQPpDspfEECqKIlBWKA2VdaOsC8A/glY/64mbWeSwdWZswf12Ca7cQ5n10JrAB6Zswfd6ap76vRaOKAOICQkJD9JR5mOxZCCF/z121YLpeL8v37yZhyK66qSZ58QaPXEz15MvrzzkN/XmcKN2+mYO0nbtvXDw4em/cSRW8vdds++M67OTT+bj7fe4pN5izyS9zPTg4QG2bgp3o1m6WkRtvSFk4u2rpzLZsmUEiQX5zr/Jn1781vh6uigoozZzhw2WjFMffato2g2Bg0Gk3A34kQaOS3oKGz7W/srda+k0KItkYCzD5QVeqiBLg1Y/64FbWe/w9wQcb8caMaec0CoDfwPXAnUApsAP6SMX9cw3up3dBoNM8CzwDo9XpsNt9NViWEEGr46zYstbfm6RMSwG7HYSvHeSanSZ/BraAgQvv1w0Xl70nprt2qJ+UpLLNz4bMbFd9iQLdo+neLoV/XaHq2D+ORj5pXUkOynluenFyIs50E+cW5zh9Z/97+dnibTSp3Iojmkr+xd+S3TJzrAiXAbE42FQKqAsImizlSTbuWDDB3Bk4AozLmj9ta6/mngdsz5o/r08hrPgcuA74C/gpEA/+mMgt6ktr3lgxmIUQg8fWBlbcnU0rtNUYjoRdfjP3ECWwnTkB5uc/GWq12LboBz28it/i3C3+NTQroDXclNZqT9SwBad+SkwshhDg3+bvuaVOySeVOBNFc8jcWQqgVQAHmu9S2NVnMb6tpF+gB5o3ASKBjxvxx1qrnrgS+qHquYYFPBVKDWQhxtvH2ZMqb9i6Xi/Shw3Dk57ttrwkOJvau336fct9+W7Fkh/F8E9ETJhJ13The+jaT9zbtdTsp4JCLEkmIDWPXsTz2nizAVuH03HeQljnXJNMnPoKk+AjahRsoKK1g8sLtXmU9S0BaCCGE8B9/1QtuTjap3Ikgmkv+xkIITwIlwOwPQS34XmcAB5WT9NUWD5xy85pM4ER1cLmKuer/3QCvA8xCCHG2iZs5k6LUVLcnU3EzZjS5vUajIXrSRI8B6dg7ptLh0Ud+e8JR4bE9QPmvZrJ+/RvZL73ElBEjGfn9z8QV5dYsj7YVMzl9CyPOpNH/T8uJ6RgHQEGpnd8957mkRnmFk+c+/bXmcUyonmC9jkxr40HvtKwi3kg9WCfr2VpibxCQzim2sTD1EJst2T4PSAshhBDnGn8E4XRRUSQsW9bkbFJ/BwYl8Hj2k7+xEOJc1RqT/O3OmD/u3lrPpQGr3Ezydy/wKtChuuZy4pzPLge+BOIz5o/L9nYMksEshDgbeXtrnj9nf/bU3tCrFx0ee4zCLz6n4IuNqiYnrJ+BrVRSQ6/T0D7cyEk3AeXGXmMI0jJzZHc6R4dwXnQIG/Zk8tGPx92+vn4ZjsYC0tXU1oUWQgghhG9JNqkQQohAEogZzOZkkwGYC9xKZTJvnRNXk8Ws6tbclg4w3wK8CzwIbAPuB6YDfTPmjzuSOOezdwAy5o+7s6p9OJUZy98Bz1JZg3khYM6YP25yU8YgAWYhxNnO25Mpf8z+rKa9o7CQgs/Wc+r558HhcPveuuhoen/3bc3jeRvMHktqTB17IXOuSaawzE56dhG/nrTy5Jp9hNtK3L6myBCqen0BBAdpuWdEd0INOkINQaSmZZOadsZte3d1oYUQQgghhBBCnBsCNMD8InALMA94BXgSSASmAE+ZLOaFavpp0QAzQOKczx4E/gR0AvYCj1bXZE6c89kWgIz54y6r1b4PlRP7jQDygDXAnIz54wqb8v4SYBZCiObxZQDbWVbG/n79FfvQd+tG2JAhlZMPJvbkl3sfplPeyQbtMmM60//j30pqVBvx5FrmfvEKiYUNKytlRMTzl8t+T/fuHTmRX0pWQcNJDb2deLB++7gwAzufGuu2vbc1m/1d41lqSAshhBBCCCGEbwVogPkw8IDJYv7cnGwqBPqZLOaD5mTTA8DlJot5kpp+WjzA3NokwCyEEIElbegwHHl5Pusvdto0Osx6DHQ6NBoNAGsffILeX69x+5r0MeO5YcHfAbBVOBn8wpc4rFa3Gc/lIWH07xpDib2CovIKMs6UKGZI3z0sgav6dmJQYgx6ndbrms3+nnRQakgLIYQQQgghhP8EaIC5BEg2WcxHzcmmTOA6k8W805xs6g7sNlnMkWr6kQCzEEKIVpWdkuJxUsDI68Zh6N6dku9/oHTXLlzlDTOM3dLp0Oh0uGw2j8004eEkLnsXfefO6CIjSVn1A33nz3ab8fzrnBRmTRxU85xShvTskQ/VlOGIDA5iWM927Dqez6lG6kQ3VrO5KTWevQkYSw1pIYQQQgghhPCvAA0wW4C7TRbzd+Zk0zfABpPF/Hdzsuk24BWTxRyvph8JMAshhGhV3kwiWFFQQPrFQ/w6Hm14OK4gPa5891nVIdffSOeHH0AbFoY2LIxPH/srvTe7z5BOHXA1qwff1GgAtzEDukUzIqk9eq2GIJ2WrWmn+fZQjtv2dw1L4Mlx56PXaQHvA8bzNphZmHrIbf9SQ1oIIYQQQgghmidAA8zzgCKTxfyCOdk0CfgAOA6cB7xsspjnqulHAsxCCCFanTeTCCqV1NBGRNDl36/hqnCAowJXRQUn//RnnK2479fGxNDn2+0czSnhK0sWL3xmpsLZ9N9fdzWhQw06IoP12Cqc5Ja4z9ruEGEkNsxAic1Bqd3B6cK6WeH+riEthBBCCCGEEOeaQAww12dONl0CDAPSTBbzOrWvkwCzEEKIgKI0iaBSSY24mTPoMGuWV6+JnT6d2DumYj95ElvGETL/8hfvB66gz+5daI1GyuwOkp/6vM6yxgLGF3WNwukEm8PJ/lOFijWePVEzSaFS/9NHJDL2/I4MTGhaDWkhhBBCCCGEOJe1hQBzU0mAWQghRJviTUmNpr5GKUtaFx1N4kcf4iwuxllSwrH7H8BZ5Ln8RdSNNxJ982RCBgxg4N++xJaX7zaga4yJrpMxrFTj+fkrH+HlaSOwltrJKSrnqbX7FAPGs6/sTWSInmC9jpRVP/Lsl/9SXUP6kh5x/HLcyqkCdTWk6/M241kypIUQQgghhBBtXSAGmM3JJqPJYi6v+vd5wL1AKPCJyWL+Rm0/EmAWQgjR5nhTUqMpr/E2S1qpfW2GHj3Y1ftiIv7vK7oWnW6wvLFJBNc++AS9v3Zf4zl9zHhuWPD3msdKAem/X/Uo3/ztRlwVFTisVjY99gyJO75y2/+WAVezetBNpGc3HkSvnyHdWM1mbzOeJUNaCCGEEEIIcTYJpACzOdnUB1gNJAO/ALcDm4BIwAmEAZNMFrP7E9FaJMAshBCiTVMqqdGU13ib8eypvaFHD6LGj6dg3TrK9+9XNT7jiEuJHjUSZ1EhjoJCct97D2zuayqj1xNx2WVoDAY0ej1Hd+wiKjPDbXNbSBjBeh3OggJV49FERNDnu285bi3na0s2f/vsV4ylxW4zpB1h4SyYOpCBCTFEBuu9nnTQ2/b1ScazEEIIIYQQItAEWID5UyAEeAWYAowEvgJmVjX5NzDQZDFfoqY/CTALIYQQjfA2S1qpvcvlomzPHvJXrCR/xYpW+ETNo4mMJGLEcAxDhzN+i5W537+jWFJDq4HkjpFoNbD3pPtgdu2MZ5fLxQufmVn8f4dVta/W3IxnCUoLIYQQQggh/CnAAsyngbEmi3mXOdkUAViBwSaLeWfV8mTgO5PFHK2mPwkwCyGEEAq8zZL21N5ZVsb+fv0V+wjq1pWgyCh0kRGU/PAjLrvdbVuN0UjUDTfgsttxlpZS+MUXiv13mjePoPbt0cVEc/Se6TitVsXXqLWi92iW9h2H2kMMrQY6RARTbKugxObA4fT8whC9jn9N6YepUyTnRYdQWFbRpIzn5gSlpY60EEIIIYQQwhsBFmB2Ah1NFnN21eNC4CKTxXyo6nE8cNJkMas6iQny20iFEEKIs4S3JTg8tdcGB6OLifE8iWBsLEkbN9Y8VqrxHHvnHXVqQitOUhgbS/T4m2oex0ye5LH/6JtvxpiURNH/fUPJju9xlTWc3K+2G07u5I8f/oudR3PZfiCnQTZy/ZrNTheNThjorn2p3cG97+4EINwYRJhRR1ZBeaOvTcsq4o3Ug41mPNcOShscdnKKYWHqITZbshsNSksdaSGEEEIIIcRZpH5mT5OzkCWDWQghhGhh3k4i6G1NaH/2X2G1kj5EuQxX3H33EXHF5QT37cvAF77ClpfvtmazMzyCZ64/n1BDZbD4jx/uwlVQ4LZ9qTEMjQYqFDKdq2k1kBBXWXc6RK8lxKDjeG4pOafOuH2PqWMvrBOUbuk60kIIIYQQQoizSwBmMG8CqjN1rgFSgZKqx0bgCrUZzBJgFkIIIVqYtwHj6teorQnt7/6VMqRr07VrR0a3ZIzmPXQobViGIyMinl/npDBr4qCa51JW/UDf+bPd1nj+dU4KD9/Yn4PZxfxyPJ85q/fUaVM/47kx4bYSXv7mP27f48+XPsT5fbqQGBdGQlwYPx/NY+OvDdtWq18Xet4GMwtTD7kdU2N1pJtDSnAIIYQQQggR2AIswLxUTTuTxTxNTTsJMAshhBCtwNtJBGtTUxPan/0rZUgbzzfhzLdiP3nS4/tU+//27jw8qvrs//j7O5PMJJmEbOyL7JCIVBZXFBEr2or7+rjUKnV73PpYi2LrvlL1qXVtsb8+7trauotLwQIiIiKuYMIWAsgaQvZtkpnz+yOLwySzZrINn9d1cZE55z5nvlOvU5IPN/c3+aSZ9L/qCuxpadjS0tj22JNUPf9swPrUS2YxZM7slteT7lkQtEPaSkvjtyeMpbbeS029h8raenj6Cc5Zvzjge7w6unGOdLgSbIaJB2RgtxkS7TY+KyjGWVMVcE3OzAxW3TYj4P3CCYw1gkNEREREpOfoTgFzrClgFhER6WKRbiLY1fcPp0Pa1qsX7g0bqFyyhN1/ehQaGmL2/rZevRg5/13svXtjjAmr4/k3Z06moagId8Em3IWb2HLP/Tg8gTdOLHOmsvC+5ygsrqZgTyUFRft+7xCqSzpUh/TsqdfwxdwzSHL8uB1GJIGxRnCIiIiIiPQsCpjjiAJmERGR9gu3Q9pbW8vaCRM7ZA0mKQnHkME01NTh+WFrwDpb795QU4M3wj//Ry5ehKN/fwAm3b0Ad2ngLmmPK43Lpg6n3mPh8Xph3hOcuW5RwHu/Ono6C6eezc/G9ednB/VnRO9Uznt6ediBsf8IDn+xHsEhIiIiIiLto4A5jihgFhERia1QHdKhZjbbMzMZ/vpreCsr8VRUsvWKK/BWtg5aYyWhTx8aSkpCd1UnJOCaciS9fn4Sz1ZnMvZPdwTtkm6eI91QUsKa6cfjqK1uVdus1OHi/JPuanmd7LBT4/YErJ82pg/jBvZia0kNW/ZW8+3W0n22ePbvqM52Odo9gqM99SIiIiIisi8FzHFEAbOIiEjnCjWzOfvyy+h7441h12de/AvSTz2N+q1bqCvYxJ7HHw+5hgH33YtzzBgcw4djT00N+R7Y7eDxCXxtNvB6A5YnjBxFYqoLd2EhnrLWmxm25b2H/8H8tXtbjd8IV6q7OmBHdaUjhZtOHMMZkwYzID0ZiHxms2Y8i4iIiIjEjgLmOKKAWUREpHOFM7PZd6xGpPUhO6Szshjz6bKI1jR43tPUrFpF+XvvUfnJJ1AfeF5zNExiIkP++jQphx/O99vLmPn4vutra8bzhCHpDM12MSQzhVcXreGujx4LOuO50pGCMXD48CxOOLA/L63YzMamMNv3/m2N4NCMZxERERGR2FLAHEcUMIuIiHS+cGc2R1MfaYd0pO9Rv3s3G46ZFvIzZs2ahXPUKBzDhlH+3nxKXnwp5DXOnByyLr6YE751Ul1eFbAj2ZmZsc/Ii7evupnRi98OeN+PJpzIvJyfU1H74xiQYB3Pp03N4YpjRuC1wOO1ePrjjbz6xQ8t1/oH3prxLCIiIiISGQXMcUQBs4iISNcKNbM50vpIO56jeY9Iu6SDrSmhXz/sWVnU5eW1HKtN7UWt20OGu/X3KFtS+7LtvMs4vZ9FbV4+tfn5+1zbpoQEep17LgWZQ1jg7sWbW938YdlfQnY8+woWSDsyM/hSM55FRERERMKmgDmOKGAWERGJP5F2SEcqmi7pYGuy9epFzZdfsvfZ56j46KOg851jwYPBTuDv+V4dPZ1nxs1seZ3qruahpU8GDaSnTR7JCeP6MW1MHzJSHJ0641mBtIiIiIj0NAqY44gCZhERkfgWaYd0ONrbJR1sTe4ffqBg5slYdXVB15AwYABJOTkk5eaw97nn8Qb5fsYkJZE8fjy1+fl4KyqC3hegwdhxzjwZ24CB2AcNZOEzb3L05lUB630DabvNcPDgdLbsrWZPpRuIfMZzqPrma9qz6aBCaRERERHpSgqY44gCZhEREYlGR3VJe2trWTthYsi6sd983RJSh9tRbVkWdRs3sunkU6JeX1uqktN49NrH+aygmHpP4/eSwUZqHP6TYZx/2AGkJSWQlpTIi59t5u1P8gPWXzRj/D4znqMJpJuv66wuaQXYIiIiIhKMAuY4ooBZRERE2ivWXdKxnPHcVkf12iOm4C0NfH+cTlKnHk39D9twb92KFcb3SsmHHUZC7oEUZA5i7rdVXLfylbBnPIcawXHrcdcx62cHk+1ykOVy8N53O1j4+YawA2loHUr7ilWXdHu7qkVERERk/xFtwJyXk3sNcCUwrOnQGuDe3Py8+U3nDXAHcAWQCawArsnNz1sTi3WHQwGziIiISBeL9Yxn/47qSO+/7ogj8ZSWRvlpWvvP4Im8NeoYam2J1NkTOXPDEk7d9GnA+khnQt923PXcdM6h9OuVRN80J/16JTFvyUbmfVzQUufb9Qxw1bSRQbukfYUz5iNUvYiIiIjs39oRMJ8GuIH1gA34JXATMDk3P+/bvJzcm4FbgUuAtcDtwNHA2Nz8vNDz8mJAAbOIiIhIF+vIGc/R3D9UIO2aOhXHkMHUrvme2rVrsWprA9ZGw2NsbO89hFpsVGOnb9VeBlTvDVjvH0g3Cza2w0pL43/PmUCvprEdL6/YzCsrt7ZcGyqQfuD9POYtKSAQ/3p/GsEhIiIisn+J5YiMvJzcvcAtwNPAduCJ3Py8+5rOJQO7gd/m5ufNi8X7haKAWURERKQb6KgZz9HcP5JA2lNZybpDDm33+tqj3th596AZfJU6iPzMA6hwuEJ2PfuP7YDggXS1M4UB6cnYbJBgs7G5uAqvz7fR/oF0tsvBqttm7HN/35EaFeVVpPVyaQSHiIiIyH7CGFMDTPA5VGxZVnEk98jLybUD5wDPA5OBKmAjcFhuft5Kn7r5wJ7c/LxftnvhYVDALCIiItLNxHrGczT3jySQDjXj2ZaZyaj338NbW4u3poZN556HVRH4X+uZ5GR6X3E5Xrcbb3U1Jc89H9nnGzSEHyrqOaB8Z8CaV0dP5+WDT6GuwQuEHsMRaSBd6UjhtAkDmXRAJuMHpzMoI5krnvoPk5e906p+1VGn8Pz1M2I6gkMdzyIiIiLdizGmHvD9Bu4uy7LuDOfavJzc8cByIAmoBC7Mzc+bn5eTOwVYBgzNzc/b4lP/f8Cg3Py8E2O1/mAUMIuIiIhIUKEC6UhnPEdaH3KTQocT57Ch1K1fD2F+b+u12XAdfDA4HHjsCRR+u46BlUUB6xcOP5x+v/kNda40PBY8+c5X3P3RY2EH0mnuah4MEmAv//X9XHXyRJIS7SQn2nlk4Tqe/jiyERzqeBYRERHpvtrTwZyXk+sADgDSgbOBy4FjgV4oYO58CphFREREYivSGc+xngndHEh7KiupXb2a6i++YM8TT7bzUwVgs2HPyqK6xo2zqjxg2fcjJpJ3yHHklzaww22YuelTfr7584D1geZIN/MfweFy2nnqwskMSE+if3oSXq/FufOWt3Q8+9ar41lERESk68V4BvNCYDNwHxqR0fkUMIuIiIjEXqQzpDtqJnSzdUdOwVMSuOvZJCeTfekleOvq8FbXUPrKK1F+8tgodbg4/6S79jkWagSHr0SbwVlbFbD+ohnju7TjWZsaioiIyP4uxgHzf2jc3O8XTb8/npufd3/TuSQaN/mbrU3+OogCZhEREZGOFekM6VjPhIbIx3CECqTtGRkM+etfaSjeQ8POXey8887QHyxSY3Px5I6nbkwut39dxbXLXwg4UuOmYxpHcDR/Kx9qhvScaddy0pSx5A5II6d/LwakJ/Gr51Z2aMdzZ25qqEBaREREurtoA+a8nNy5wHxgK5AGXADcDMzMzc97Py8n92bgd8ClwDrgVuAYYGxufl7gjU9iSAGziIiIiPQo4QbSHTGGo1nIQDori9HLPsFq2thwzfEn4qhuvWFftLYccixTfvdrShoMP1Q38PHdj3BawbKA9W2N4AjWIR2q4zlUYFxWXc/Fjy2IelPDcAJvzZwWERGRnqQdAfOzwHSgP1AGfAs8lJuf92HTeQPcAVwJZAIrgGty8/NWx2jpISlgFhEREZG41JFjOCINpLc+8CCVzz0TsN45fTqu4cOp+eYbalevxqqri+SjhlSVmMTzl97D59UOdlbWh+x4/t3067hq5gTG9m/seE5KtPHLxxaGHRg//NpKxs2dHfD+K294gOtPn0xSoh1ngo25H+Tz0oLVYQfe0QTSIiIiIl0pliMyuhsFzCIiIiIS92I9hiOajQ0LLriQho0bW9UnjBzJiJdfaqn3VFSw7tDDovmYIRmHAzN4CFt2lTG4ak/AOv+O5yxvLfctejxgYPyXs24mq182xVVuiivdnLjsn5yzfnFY9zcGXHXBA+/fT7+Ok48aS4ojgRSHneUbi/kuf2tEHdgiIiIiXUkBcxxRwCwiIiIioXTEXOhI6kOO4MjMZPibb2LV1WLV1bHpgguxKmI3Ys9jDFt6D2WvcVCZmMzgit2MLN8RsH5N1jC+zx5GoqcBh7eB47d8gcPbELC+1p7IizknUJ2QRHViEkdt+5apO74LWO8feIfqwL7vxBv45N7Tgn5GbTwoIiIinUkBcxxRwCwiIiIisRbrjQ0jHcERqj7r0ktJP+N03JsKqduwgT2PPx72WruDepudNQdPoySpF3ucaQxY+zVHhwik95x/GadOGMTU0b1bguFI5zbH05xnBeQiIiJdSwFzHFHALCIiIiLdXTQjOCKpX3vEFLylgTukTXIK2b+ahbeinIbSMsrfeivkmpMPPRRbchI2p5PSRYuxNwTuYPba7CSPGom3spKGykqs8vKQ949EqcPF+SfdBUCqM4EZB/Zj2pg+PLloA+t3t95ssa25zf5znkPVd7ZwAuN4CshFRER6OgXMcUQBs4iIiIj0BB05giPSDumvJx+OsypwCFyX2osJX6xoeR1qU8PUS2YxZM7sltehAm8cTlKnHElDURH1u3fjKSoKXNvkvqueYNmuWgL9uOO7MSDA2ZMHM+uo4dhsYDOG/7e0gFe/+CFg/VXTRgad8xxpx3B1RRUpacF/5owkMG5vQK6OZxERkdhSwBxHFDCLiIiISE8T6xEckXY8RxoYR7KpIUQeeK894ki8paUB6wHsWVkknnwayw+cyhs7LVYWlpDqrg64MWClI6X15wpS78zMYNVtM/apj7RjuGRnMR/f/TD9li0kva6SMmcqu446nmNu/y2Z/bNb3TuSwPiB9/OYt6Sg5XU4Abk6nkVERDqOAuY4ooBZRERERCSyjudIA+No7h9J4B0qkN6HMSQfdTR31Azhv9Z9FHBjwNlTr9knZA61keDsqddw0fEHMWNcPyYMyaSytiGiALhkZzFfnXEuA0q2t6rfkTmQiW+8uk/I7B8Y+5s+tg9Hj+5DeU09ZTX1vPz5FhzVlQED8oSMdL66bQbGGKD7jwQRERHp6RQwxxEFzCIiIiIi+wqnQzrSkR0def9QgfSABx+i4v33KX3tNTzFxUHft9nbB/6US599GK9l4fV4eHHWTZySvyhg/aujp/PMuJkApCcn0jvVwcaiH3/OaKtj+KYTx1Jd76GytoGPf3s745e+E/D+CyecyDc/v4iK2nrKaxtYu7Mcr8+Pbv739xdOQE5qGqP7pzGmbxpb9lazvCDw/1ahRoKIiIhIcAqY44gCZhERERGR9ol0ZEdH3D+cQNpyu6lYuJCSV/5O9cqVQe9nASaCNVYmpTH7wj9QWFzdcizYSI0qRwoYWmZC//2920l3Vwe4+74bFYZz/0pHCsN6p5CR7KBXciJj3nyWs9aFF5C3xT/AznY5Wo0EERERkfApYI4jCphFREREROJLqEDaW1vL2gkTY/6+KVOm4D7wJ3zZawh/WF3Dfcv/2mbH8Oa0vsw76FQGV+1hVOkPjCnZ2madv89OvoSacRNg6HCe+fdq7v7osYAdyfefeAOLZx9L7Zo11Hy3mp2PPY7d0xDw3nXOFDY9+Qr5FR7yd1SwZF1RyAA7764TSXYmtHk/bQooIiISnALmOKKAWURERERk/7PuyCl4SkoCnrelpTHoj38EYzA2w60IHdcAACAASURBVA/X/xpvZet5xIF4AVsM1hlobRWJybj27g5Y40524aiJ8Occm42k8QfhOuxwbs33cObX7zK0ovV7NI/UyOiXzVmTBnPWpMEckJ0Sd5sCKiQXEZGOpIA5jihgFhERERHZ/4TaGDD78svoe+ONYde7ph5NQu8+1Hz5Je7Nm8Nag71Pb5LHHUTSuHF8+/EqBn73WcDayux+9KqvwVteHta9fTmGDqV+xw4stzviawPxH6kx+YAMfiitYVd5XavacDYF7C5hbryF5CIi0n0pYI4jCphFRERERPY/oTYGHPrii/tsJhhJvXv7djYe99OQaxj7zdctozxKdhbz1RnnMqBke6u6HZkDmfjGq2T0zcRdUEDVihXsuufekPcf8vQ8kg8+GHt6esiAPPMXF5Fy2GFUr/ic6hUrqFu/Pui9vekZvH7b33jzq20UV7UOrtva1NB/U0DfMLeivIq0Xq4uDXPLqus5Z96nrNvVulM9nJBcREQkEgqY44gCZhERERGR/VM4GwNGWx9qBIc9K4sxny7b51jJzmKW3v2/9F22gPS6Ssqcqew+agZTb7+RzP7Z7bp/JAF5uDOqM847D9dJJ7EiZTBX//1rnDVVAWc2u5NTuXLaCIZluxjex0Vvl4Pr/rqEycveaVW/6qhTeP76GTEPc0N1ST/wfh7zlhQEPN9WSC4iIhItBcxxRAGziIiIiIiE2hgw0vpIR3D4q66sJiU1Jab3j2VA7svepw/vpIxkUtE6+lfvbXW+eWZzpePHz5PqruahpU8G3KTw+zkPc+NZhwR8z3BHakQy8mLi3f+mpLo+4L2yXQ5W3TYj5HuKiIiEQwFzHFHALCIiIiIisRbpCI7Ovn97A/LkyZPwVlRSt25dWOtdOngC6yYdy86yGmrcHk7Y/DnHbP82YP0bOcdx3l//wIjeLowxQOTzkf1HXviO7RjTL5WnLpzE2p2VrCzcy2cFxeTvrAj5OfLv+VnAYLu7zJEWEZGeQQFzHFHALCIiIiIiHSHSERzd6f7hBth1GzdS/sEH7H7iKWyWt13v6avU4eL8k+5iYHoSR43qzaShGfx16SYKilr/7BZoPvID7+fx0oLVAcd2+HZUh+v0CQM5c9JgjhrVG7vNtGtTQAXSIiL7NwXMcUQBs4iIiIiIdLRIR3B0h/uHG2CHO7M5UheceBslyW0H5f6bCB42PItDhmZSVddAZZ2HqroGln+7ibmLnwg4hmP21GuoS3bxk8EZHDIsk83F1XywemfA9/DVv1cSPz+oP4vW7qawuLrV+UChd3sCaRERiS8KmOOIAmYREREREZHgQgXY4Ww6OHrZJwAYY/h68uE4q8qDvqeVlMwPR53A6yOm8kFR49zmSLqRZ615l3PWLw54/1dHT+d//vEn0lMcQGP4e/FjC9rcePDzI0/m1KNymL96J99sLQ267mb+mwL6j+zwFSiQ9tXRHc/qqBYR6VwKmOOIAmYREREREZH2iXTTwa0PPEjlc88EvqHdDh5P49cJCSzuN44xpT8wsKq4VWlzN3LOqEGkJdkYVLmHIUWbOfz1eTg9gTftq0lwkvPCMziHD8eekYGnrIyCCy6kYePGVrUJI0cy4uWXsKens2F3Ba99uY2/LNmI74/P/h3PdpvhqFG96ZvmpE+ak2+2lvLpxuKA9f6BNLSv4zmcwFgd1SIiXUcBcxxRwCwiIiIiItI+kW46GCrMPeCpJymfP5+SF14M2hndbGPWEMaP6EdtXh7eKH6+s2dmYhwOGna1HqfRzDckr633kHPbBxF3VQerJ60Xb15zFMOyU0iw26LqeI4kMO7sjmp1SIuI7EsBcxxRwCwiIiIiItJ+kW46GE69t6aG0jfeYOc992Ii+FnVJCdjud0/dkG3WWQggnva0tIYvWQxtpTG4PjoW9/i9x8+EnDG8+0/vZ6ZU8ZSVFHHrrJa8jds46GlTwadCV3pSMFhtzGij4sGr8WG3T+Gv6E6niMNjB94P495SwoCft5YdFSrQ1pEJDAFzHFEAbOIiIiIiEhsRbrpYLD6cDcRzDj/fJIP/gnJ48bhGDGCokceCTm2I2vWLNybNlG3di0777o79EJtNpxjxpB88MHkf7OO/vlfBSxdf9wZnPrU/S2vHzzzWk75/qOA9a+Ons4z42bucyxYx7M7JZWLDh9K7zQHvVOdLMrfzfs+mxT6O23CQE6bMJCK2gbKaxu4f34eNfWBA/jMlES+vG0Gxhgg8gC7vR3SIiLxTgFzHFHALCIiIiIi0r2Fs4ngmE+X7XMs0rEdod4jYjYbif37462rw6qtxVNVhQlS3pDoxHnDb9nmzCCfVP6ycgdzl80L2fEciH/Hcyht1Scn2hia7WJIVgo7y2r5bltZwOv9O56j6ZAWEdmfKGCOIwqYRUREREREurdINxFsFsnYjlDvkXnhhbiOmkLNN99S/dVX1Hz+efQfKAwWBA2kXxt7HPmnXMyeyjqKKuoornKHnAltgF7JiaQ6E9hZVktyXVVEM6SDMUB2qhNngg1ngo3NxdV4guQL2S4Hq26bEdF7BKMZzyLS0yhgjiMKmEVERERERLq3SLuR2xJqbEesO55NSgr9bpqNSUrClpTEjltvw1vZelxEi4QE7JkZeIr2BP0cLetNSGTADb8madxBJB00jmlzPwo6E/q+E29g6T2ntoy8ePi1lYybOztg/ZJr72XKhBFs3VtNQVElr325bZ+aWHRIf3nr8WSltv3fJJzAuL0znhVKi0hXUsAcRxQwi4iIiIiIdH+RbiLY0e8RaVd1uPXe2lrqNm2i8IwzI1p7XUoqzurAAfamKSdy/N2zAQssix1/fprq1/8VsD71klkMmTO75fWkexbgLikN2PFMWi/uOOVA3B4v7gYvD36wFntVRdAO6aREG8fl9OXknwxk+ti+uBu8YQfG0c541saDItJdKGCOIwqYRUREREREepZINxHsiPeItOM55h3SDgcJ/fpRv3VruB8pIiYlhSFPPI5j5CgS+vbhf1//ImjH8/dzHubGsw5pORaqQ/qmqddQ4TOGIznRRoLdRkVtQ6t6/8C43uPl3nfzeG55YUuNf4d0WzOe/UNp32vC2XhQHc8iEksKmOOIAmYRERERERGJRqRd1R3RId1QUkL1V1+z7eqrY/75mtl69QJnEt6i3QFrko6fQfZpp2DVubHq6ih+513cny0PWJ9w4S9ZecIFvPvtdj4rKMYbIopIT04k0W6joraeugYvQNCZ09XOFKaM7M2A9CQGZCQzKCOJj9ftYcmqjQGvuWjG+DZD6c7qeFaALbJ/UcAcRxQwi4iIiIiISHtF2lXd2TOhbenpDHvlFTBgjGHTeefhLSsPvECbDbze8D5MFOwZGYxpCqCLKuqY9tAiqt2elvOhZjynuqt5aOmTATukZ0+9ptVGhaGuufP4X/OX/z6WQZnJ9HY5qahtiGoMR7OOniOtQFqkZ4vngDmhqxcgIiIiIiIi0tNEOrIjVL09PZ2hL74YdsdzxllnBu14zjz3HJwjhv/4+pxzgndI/2oW2VdehXtTAbV5+ey8/fbQH8rpxJaUhElMxLMn+GaFntJSNv/qMjJOPx3XtGOpdnuCdiRXOlK4/4yDyHI5SUtKYOlv72gzKAYYVrGLCwoWU/+rq9lRWsuOshp+KKnhtDXvBr3mpNULOOMpBwAOu42kRBvlbYzsAFi3q5I/L9nYro7ntuZIF1e5mbekgEX5u9sMsNVRLSI9gTqYRURERERERLqZ7j4T2p6VxZhPl4Vd78uWksInvXMYUbSJ/jWtrylM68f9J97A0ntPw1NRgbtwM+svupjEupqA97SMIbFfPzCmqWvbRt227dgInHmUOlycf9JdAc/7d1Un2g0XHj6UUX1TGdknlb5pTv77pVUhO54ty6Kuwct97+XxwvLNAd/Pf450tBsbNuvojmoRiUw8dzArYBYRERERERHpgbrDTOhw61OnT8fyNFC17FPweALWNatJzyI1weApLg5Z2x5Z/3yDoqwBFBRVcf3fvwrZVR2J9OREEmyGitoG3J7Q40fsNsO0MX3o1yuJ/r2S+HLLXpasC9wZHmhjw/Z0VDfTJogisaeAOY4oYBYREREREZF409UzocOtb9izh/L33mPXHx4MK2gOl83lou+cm8GywAIsL7v+8BBWTXXIax2jRpJ27LHcvCGBc1e9GXBm8++nX0fO6EFs2F3J7oq6dq851NzpUNKTE1nxu5+2hLyRBMaWZXH7W2t44bPwO6qb30MdzyLRUcAcRxQwi4iIiIiIiITWUR3S3tpa1k6YGPL9+91+G0mjR5M4dCh7n3uevX8Lv6MaQndV43CA2x1yHc3WH3cGpz51f+O9K2o57L6P9r1dG4Hx3DPHk+Vy0Cs5kSue/wJveXnADmmPK42zJg9iZ1kdO8pqWLM9yKaMTQwwvI+L3AG92F1ey8rCH0eO+K9n8tBMBmYks2lPJZuKqqhy7xvw+9cn2AwXHn4ABw1K5yeDM+iT5uS/nl4edcdzZ1BXtXRnCpjjiAJmERERERERkcjEukM60hnPkXZUh3PNAc8+i7uggIpFi6lcvBh3QUHQz2TS0xn72XKMMQBMumcB7pLSgIGxMzODVbfNaLn+4ddWMm7u7IAd0t/PeZgbzzqk5dikexawt+rHADycjudIR3xEUp9gMzR4A2dIbXU8++qo8Fdd1dJTKGCOIwqYRURERERERLpWpDOeIfKO6kiuCber2tY7G9fkQ0iZPIl/lSYx5IWnwgqMvbW1bL3nfqpf+2fAe6deMoshc2a3vH7g/TxeWrA6YAB8xjG5HJfTl+93lLNmezkfr9rIQ0ufDLie2VOv4ZjJIxndN5URfVzMffVzbl/waMD6W4+7jtEjB7J6WzmVdQ2tavwD7yyXgy99AnXo+PBXc6SlJ1HAHEcUMIuIiIiIiIh0rWg6kn1F2lEdzjWhuqojZcvujd3poKGkBKumJvQFiYlknv9fOIYNwzF0KDWudL777+sZULKjVemOzIFMfONVMvtntxx78MxrOeX7j1rVNlsw+mj++2/3YWw2MIYPbr6fkZ+8F7C+eSSI12uRv7Ockx77JGTH89RRvflpbl+m5/QlI9nRrvA3HA+8n8e8JYE7z2M9R1qBtLSHAuY4ooBZREREREREpOtF05HckUJ1Vfc6/XSSc3OoXvUl1V9+iWfPnk5cXWv27GzsaWl4a2rwVlfjqajAxPD+JjmZgQ/cT+LgITgOGMK0Bxfx+w8fCdoh7TtWIz05kbKa+oD3b+9IjWp3A4fet5CqusCbRaY6E3jv+qkMyUrGGBNVx7NGcEisKGCOIwqYRURERERERLqXaDqSYy2SrmpPTQ3rJk4Kec9+v/8dCX36Ys/M5IfrrsNbHnjjPuN0kjxxIu7Nm2nY0bpruat5EhKxNwQOjD87fCarTryAj9cVUV7beqSGv2yXY58Z1RA6zPV6LT4rKOa1L7fx3nfbqan3hrX2bJeDg4dkUF5bzxeFgbvU/UPv9o7giJQ6pOObAuY4ooBZRERERERERNoSSVd1pBsVRjJ3uqGkhPVHTgm53r433YS9VxomOZmdd9yJt7J1ENrMlp7O0OefB8sLXi+bL7k0aOCN3Y5JTMSqrQ25DgBbZiZjl39Kg8fLZ5uKuej/fb7P+bY2KbxkylAOH57N5KGZOBPsAcPcYdkpHJfTlw9W72R7WeD1hLMRYjCJdsPU0X2w2wwJNsO6XRVsLAqcIcViY0N1SO8/FDDH0LA5868GZgMDgDXA/xTOnbk0jOuOBhYD+YVzZx4U7fsrYBYRERERERGRUEJ1VUe6UWGkc6c7MsAOt77Pb35Dw+4i6jZuYOusXwWsbZY1axYZZ5+Nc8RwJt2zAHdJadCZzb5SnQltbibYloMG9eLMiYMpLK7i9SV5Ad/jguMP4tSDB/LV1lJWFZbw5tfbwrp/uFKdCSy9aTqZLkfLsUgC487ukJaupYA5RobNmX8e8CJwNfBJ0++XAgcWzp25Jch1mcAqYD0wSAGziIiIiIiIiHSlaDYqjKRDuqMD7FgH3r6SDz6Yz4dOJP0/73FA5e5W5wvT+vHsebdQl5zKmu1l1HtCZ1PGwBXHjODMiYMZ2z8NgJKdxXx1xrkMKNneqr6tjRAn3bOAvVXultf+Hc9JiTbOO2QIDV6LugYP/1q1byAdqEP64MHpTB3dh4kHZDD3/XzW7w4vMI5mk8L20AiOrqWAOUaGzZm/Avi2cO7My32OrQf+VTh35i1Brnsd+AYwwNmRBszGmGwgGyA5OXltdXV1NMsXEREREREREWnRno0KQ3VId3SAHWl9qMA76Sfjadixk4aioqCfu1nqJbMYMmc2tfUevijcy0V/Cz1SI/+en+0TkEYawj/wfh4vLVgdsOP5ohnj9wl0I+3CDuWw4VkcPjyL2noPNfUeXv3iB9wNgedItzWn2pdGcPQsCphjYNic+Q6gGji/cO7Mf/ocfxI4qHDuzGkBrrsauBA4BriN6ALmO4E7ABITE3G73cEvEBERERERERGJQEdsVNiRAXak9eEE3jaXi6rlyyl74w3K33s/6PvZMzMYs3x5y+tQYa4zM6MlbLW8XtybN1N49jl4g/wrdZvLxZC//JnEQYNI6NuX0j1lEXU8P/zaSsbNnc2wil2t6gvT+vHN7Ac5dPxQlq7fw8friygIMq85Wi/MOowjRmaTaLcBGsHRkylgjoFhc+YPBLYB0wrnzvzY5/jtwIWFc2eObeOa8cBC4IjCuTM3DZsz/07UwSwiIiIiIiIi+5mOCLAjFW7g7a2tZe2EiSHvlzBwIMnjx5N00DjeLklk0KvPtDlSY2tqH3afegEnpNZQu/o7ar5bjbeiIrLF2+3YkpODboTYa+ZMsmZdit3lwuZysf3PT1P18osB65u7sKGxmzjntg9CLmPSARm4nAk4E+x8vK4ItydwB3Mzl8POkSN7c9iwTF5ZuYVNe1rnWh0xgkMjNWIrngPmhK5eQCDD5sx3Av8Afls4d+am9tzLsqxioBjA5YrL/44iIiIiIiIiEse6OlwGsKen0/fGG+l7441BA29bUhL2zMyQM5sbtm+nYvt2Kj78kCOD1A2pLGLIy482BjuRMAaaGys9nqDhMkD5/PmUz58f9u1r3n4TmgLmpEQ7WS5H0BnP2S4Hr199VMtr/wDYvz49OZGymnqq3B4W5u1iYV7rTupm63ZVcumzn3PosCzcHi/uBi//WvVD0PX/84utrQLm9ozUUCC9/+rMgHkP4AH6+R3vB+xso34AkAs8M2zO/GeajtkAM2zO/AbgpMK5M//dQWsVEREREREREZEAQgXeGWedGXQ+cq+ZM0kaN66xI3n1Guq3bAn5ns4Dc0k+aDzJPxlP0vjxlL31Fnv/75mA9dmX/YrsK6+kftt23Js2se1//ifke0TCs3cvFf9ZhOuoKdicTs45ZHDQGc/nTBu5z/VXTxvFZ99sYvKyd1rVrzrqFJ6/fjoVdfV8vG4PH68r4sM1O/GdQ+AfSH+5pZQvt5QGXK9/fXGVmz8tWMvhI3ozYUgG7gZvq5EaxVVu5i0pYFH+7jZHaiiQFuiaTf6+KZw78wqfY+uA1/w3+Rs2Z34i4D8242pgBnAGUFg4d2bwv3pqg8vlsqqCzOcREREREREREZH2iWSTwnBHaoz95ut9gu1IN0Jcd+SUoF3V9sxMRrz7Dt6qKrxVVWy++JdhjeIwSUmkHH4Y9oMnseG5l+lb1nrMR1sznj1lZRRccCENGze2qk8YOZIRL7/Usv7mERyp7uqgmw4eMTwLZ6IdR4KNJWuLcNRUhrVJYYLNkOlyUFRRF/Bz+o/UiGbG8/686WA8j8jo7ID5POAFGoPiZcBVwK+AcYVzZ24eNmf+8wCFc2deHOD6O4liBrMvBcwiIiIiIiIiIh0vkk0KQ4a/WVmM+XRZu95j98MPB+2qzr78MvreeGPY9fbsbDzF4Q/uSPvZz8i68AJMUhLG6aTkhRcp/ec/w17P0be+xe8/fCTgpoP3n3gDS+89reVYqE0K/3LWzZTZkygsDm+vMofdxhkTB5GalEBaUgIrCvayvCDw549FIB1PFDDH0LA5868GbqJxBMZq4IbmTf+GzZm/GKBw7sxjA1x7JwqYRURERERERER6lFCbFEYa/kbzHpF2PIdT762ro2rpJ1QuXUrFhx/+OPM5Fux2kidOwOZyYUtJ4Yev80jfsTlg+e5R4zno7JNaXpcuXIT7i88D1jdvUlhUUcfyjXu4/u9fx27tNHZFn3LwQAakJzEwI5llG/bw/uq2puQ2CrXpYE+ngDmOKGAWEREREREREeleIg1/2/M+4XY8R1If7piP7sSemcmY5Z+2vJ50z4KgmxQ6E2wcl9OXyroGSqvr+W5b2T7386+PVLbLwarbZkR9fXengDmOKGAWEREREREREel+Ig1/2ytUx3Ok9eGM+Rj10UKsujq8dXUUnHoa3rKygPUmOZmsX/wCb3U1nvJyyt9+O+QaE0cMx9hsWF4v9QWbQtanHn886SefTOq0Y/jD4sKgmxReNGP8Ph3Gk+5ZgLukNGB9gyuV43P7saOslu0l1ewoDzzfuVn+PT+L243/FDDHEQXMIiIiIiIiIiLdW6Thb3cQ6xnP/vWRzqkOVe/LpKTgPHIK21esIquy9TVtbVIYasbz93Me5sazDmk5NvHuf1NSXd/y2r/jWR3MPZetqxcgIiIiIiIiIiLiq6eFywDZl1+Oc/SoNs85R48i+7LL2lWfcdaZQd/f/3yoetfRR5NyxBFgs2FVV1P70cI2w2WAASXbqX7kQWq//566DRtwb9nCOV+922a4DDCsYhfnrl+0z7FzDx1CqruaWWve5ZX37uCtd27hlffuYNaad0l1V3POIUOCrle6L3Uwi4iIiIiIiIiIxEBHzXhuro31JoX29HQaiooo//Df7Jo7FxoaYvC/QiNbairD//VPEocMwdjtlOws5qszzmVAyfZWtW11SMebaDuY83JybwHOBMYCdcBnwC25+XmrfWoMcAdwBZAJrACuyc3PWxOLtYeigFlERERERERERCTGYj3jGXrmJoXG6cQxcgSW2417w8aAdf4jQeJNOwLmD4G/AysBA9wNHAkcmJuft7ep5mbgVuASYC1wO3A0MDY3P68iJh8gCAXMIiIiIiIiIiIiPUynb1KYmcmId97Gcrux6uvZdM65eMvLI1pzMP4zpOONMaYGmOBzqNiyrOJI75OXk5sKlAGn5+bnvdPUvbwdeCI3P+++pppkYDfw29z8vHntX31wmsEsIiIiIiIiIiLSw0Q6pzpUfcgZz2efRULv3iQOHIhj6FAyzz0naH3WrEsZ8c7bDHrkj2RfcXnI9Xn27sVbVxeyrgdLoLG7uPnXdVHeJ43GTLf5bwOGA/2BfzcX5Obn1QAfA1OiXWwkFDCLiIiIiIiIiIjs52K9SWHvK6/EOXo0vX7+c/r+5jfYMzODvr89K6tHbu4YgQYa5yg3/3o8yvs8CnwNLG963b/pd/8dF3f5nOtQCphFRERERERERET2c/b0dIa++CLZl1+GPSur8VhWFtmXX9ZqA8Fo6kN2SIc4Hwcsy7LW+fyKZjzGH2mcrXxWbn6eJ/ZLjI5mMIuIiIiIiIiIiMg+Yj3j2VNWxuaLLqJu/YZW55yjR7UZSseTaDf5a5aXk/sI8F/A9Nz8vHyf4yOAjcBhufl5K32Ozwf25Obn/bIdyw6LOphFRERERERERERkH7Ge8Rxpx7P8KC8n91HgfOA433C5ySZgJzDDpz4JmAp82hnrUweziIiIiIiIiIiIdKpIO6R7umg7mPNycp8EfgGcDnzvc6oyNz+vsqnmZuB3wKXAOuBW4BhgbG5+XkV71x5KQke/gYiIiIiIiIiIiIiv/Slcbqerm37/yO/4XcCdTV8/CCQDTwKZwArghM4Il0EdzCIiIiIiIiIiIiIdqr0zmLszzWAWERERERERERERkagoYBYRERERERERERGRqChgFhEREREREREREZGoKGAWERERERERERERkagoYBYRERERERERERGRqChgFhEREREREREREZGoGMuyunoNncoY4wVqunodncQACUADsH/9hxaJH3qORXo+PcciPZ+eY5GeT8+xSM8WD89wsmVZcdnsu98FzPsTY8wYYC0w1rKsdV29HhGJnJ5jkZ5Pz7FIz6fnWKTn03Ms0rPpGe7e4jI1FxEREREREREREZGOp4BZRERERERERERERKKigDm+FQN3Nf0uIj2TnmORnk/PsUjPp+dYpOfTcyzSs+kZ7sY0g1lEREREREREREREoqIOZhERERERERERERGJigJmEREREREREREREYmKAmYRERERERERERERiYoCZhERERERERERERGJigJmEREREREREREREYmKAmYRERERERERERERiYoCZhERERERERERERGJigJmEREREREREREREYmKAmYRERERERERERERiYoC5jhljLnaGLPJGFNrjFlljJna1WsSkbYZY24xxqw0xpQbY4qMMe8YYw7yqzHGmDuNMduNMTXGmMXGmHFdtWYRCa7pubaMMU/4HNNzLNLNGWMGGGOea/rzuNYY870xZprPeT3HIt2YMcZujLnH52fhTcaYe40xCT41eo5FuhFjzDHGmLeNMduavn++xO98yGfWGJNpjHnBGFPW9OsFY0xGp36Q/ZwC5jhkjDkPeBS4H5gIfAq8b4w5oEsXJiKBHAs8BUwBjgMagIXGmCyfmpuAG4HrgEOB3cACY0xa5y5VREIxxhwBXAF863dKz7FIN9b0g+gywAAzgVwan9fdPmV6jkW6t5uBa4DrgRzg102vb/Gp0XMs0r2kAqtpfF5r2jgfzjP7MjAJ+FnTr0nACx24ZvFjLMvq6jVIjBljVgDfWpZ1uc+x9cC/LMu6JfCVItIdGGNSo6I0jwAAB31JREFUgTLgdMuy3jHGGGA78IRlWfc11STT+Afrby3Lmtd1qxURX8aYdOBL4DLgDmC1ZVnX6jkW6f6MMfcD0yzLOirAeT3HIt2cMeZdoNiyrF/6HHsOyLYs62Q9xyLdmzGmErjWsqxnm16HfGaNMbnA98DRlmUta6o5GlgK5FiWtbbzP8n+Rx3MccYY4wAmA//2O/VvGrsjRaT7S6Px/59Lml4PB/rj81xbllUDfIyea5Hu5mka/0J3kd9xPcci3d/pwApjzD+MMbuNMV8bY5r/ggj0HIv0BJ8A040xOQDGmANp/BeC7zWd13Ms0rOE88weCVTS+K/3my0DqtBz3WkSQpdID9MbsAO7/I7vAo7v/OWISBQeBb4Glje97t/0e1vP9aDOWpSIBGeMuRwYBVzUxmk9xyLd3wjgauARYC4wAXi86dwT6DkW6Qn+QGOzxvfGGA+Nmcd9lmU91XRez7FIzxLOM9sfKLJ8RjRYlmUZY3b7XC8dTAGziEg3Yoz5I3A0jf+8x9PV6xGR8BhjxtK498HRlmXVd/V6RCQqNuALn5FyXxljRtM4v/WJwJeJSDdyHnAxcAGwhsa/KHrUGLPJsqy/denKRETimEZkxJ89gAfo53e8H7Cz85cjIuEyxjwCnA8cZ1lWgc+p5mdXz7VI93Ukjf+KaI0xpsEY0wBMA65u+rq4qU7PsUj3tYPGGY6+8oDmjbL157FI9/cQ8LBlWX+3LOs7y7JeAP7Ij5v86TkW6VnCeWZ3An18Rlo1z27ui57rTqOAOc5YluUGVgEz/E7NYN95NCLSjRhjHuXHcDnf7/QmGv9gnOFTnwRMRc+1SHfxJjCexk6p5l9fAH9v+nodeo5FurtlwFi/Y2OAzU1f689jke4vhcaGK18efsw+9ByL9CzhPLPLgVQaGz6aHQm40HPdaTQiIz79EXjBGPM5jd8oXwUMBP7SpasSkTYZY54EfkHj5kIlxpjmOVGVlmVVNs2P+hPwO2NMPo1B1a00bmTwcpcsWkT2YVlWKVDqe8wYUwXstSxrddNrPcci3dsjwKfGmN8D/wAmAtcDv4OWeY56jkW6t3eAOcaYTTSOyJgI/AZ4HvQci3RHxphUGvcxgca/DDrAGDOBxu+jt4R6Zi3LyjPGfADMM8Zc0XSfecC7lmWt7czPsj8zPjOwJY4YY64GbgIGAKuBGyzL+rhrVyUibTHGBPo/4rssy7qzqcYAdwBXApnACuCa5uBKRLofY8xiYLVlWdc2vdZzLNLNGWNm0jhPfSywhcbZy483bxyk51ikezPGpAH3AGfQ+M/jd9D4r4nutiyrtqlGz7FIN2KMORZY1Map5yzLuiScZ9YYk0njxrynNh16G7i2qQlEOoECZhERERERERERERGJimYwi4iIiIiIiIiIiEhUFDCLiIiIiIiIiIiISFQUMIuIiIiIiIiIiIhIVBQwi4iIiIiIiIiIiEhUFDCLiIiIiIiIiIiISFQUMIuIiIiIiIiIiIhIVBQwi4iIiIh0IGOMZYw5u6vXISIiIiLSERQwi4iIiEjcMsY82xTw+v/6rKvXJiIiIiISDxK6egEiIiIiIh1sIfALv2PurliIiIiIiEi8UQeziIiIiMS7Osuydvr92gst4yuuNcbMN8ZUG2M2G2Mu8r3YGDPeGLPQGFNjjNnb1BWd7lfzS2PMd8aYOmPMLmPMc35ryDLG/NMYU2WMKfB/DxERERGRnkoBs4iIiIjs7+4C3gYmAE8DzxtjDgEwxriAD4FK4DDgDGAK8H/NFxtjrgTmAc8APwFOAlb7vcftwFvAwcA/gP8zxhzQcR9JRERERKRzGMuyunoNIiIiIiIdwhjzLHARUOt36knLsm42xljA/7Ms63KfaxYCOy3LusgYcznwMDDYsqyKpvPHAouA0ZZlbTDG/AC8aFnWnABrsIC5lmXd0vQ6ASgHrrAs68UYflwRERERkU6nGcwiIiIiEu8+Bq7wO1bq8/Vyv3PLgZlNX+cC3zaHy00+BbzAgcaYcmAQ8FGINXzb/IVlWQ3GmCKgb3jLFxERERHpvhQwi4iIiEi8q7Ysa0MH3DeSfwpY38a1GlcnIiIiIj2evqkVERERkf3dEW28zmv6Og8Yb4xJ8zk/hcbvo/Msy9oNbAN+2uGrFBERERHphtTBLCIiIiLxzmmM6e93zGNZVlHT12caY1YCi4GzaQyLD2869xKNmwA+b4y5HcikcUO/1326ou8DHjHG7ALmAynATy3L+t+O+kAiIiIiIt2FAmYRERERiXfHAzv8jm0DBjd9fSdwFvAYUARcalnWSgDLsqqNMScCfwI+p3GzwLeAXzffyLKsPxtj3MCNwB+AvcB7HfVhRERERES6E2NZkYyOExERERGJH8YYCzjHsqx/dfVaRERERER6Is1gFhEREREREREREZGoKGAWERERERERERERkahoRIaIiIiIiIiIiIiIREUdzCIiIiIiIiIiIiISFQXMIiIiIiIiIiIiIhIVBcwiIiIiIiIiIiIiEhUFzCIiIiIiIiIiIiISFQXMIiIiIiIiIiIiIhKV/w9bIdK40nk84QAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1440x504 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RjTIGveF10MI",
        "outputId": "0d4787d6-cbb7-4e41-b084-e67dcd3d78a7"
      },
      "source": [
        "y_true = valA0.y\n",
        "y_pred = clf.predict(valA0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model\n",
            "[[4963  475   94    7    3  468]\n",
            " [ 174  450  604   13   11  420]\n",
            " [  67  271 3617  428  237  415]\n",
            " [   0    4  131  231  310   28]\n",
            " [   0    0   10   38  364    2]\n",
            " [  46  169  173    1    0 1218]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.83      0.88      6010\n",
            "           1       0.33      0.27      0.30      1672\n",
            "           2       0.78      0.72      0.75      5035\n",
            "           3       0.32      0.33      0.32       704\n",
            "           4       0.39      0.88      0.54       414\n",
            "           5       0.48      0.76      0.59      1607\n",
            "\n",
            "    accuracy                           0.70     15442\n",
            "   macro avg       0.54      0.63      0.56     15442\n",
            "weighted avg       0.73      0.70      0.71     15442\n",
            "\n",
            "0.5972200545917337\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pvq5WHEmF5Cp"
      },
      "source": [
        "# Try loading the checkpointed model\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nhvq0RD94Tkg"
      },
      "source": [
        "# Using first group trained model and prediction on the final test set (Submitted model predictions)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ENk570_b5Tb4",
        "outputId": "817c2bb9-b5fd-41fb-aa71-74e0628c2f4e"
      },
      "source": [
        "# Load the Channel 0, batch_size 1024 model\n",
        "# starting wandb init\n",
        "wandb_run = wandb.init(name = \"AttnSleep_TL_new1\", project='cnn-fulltrain', entity='sleep_hacking')\n",
        "\n",
        "# batch_size\n",
        "lr = 1e-3 #change\n",
        "batch_size = 1024\n",
        "n_epochs = 100\n",
        "\n",
        "channel = 0\n",
        "save_path = \"/content/drive/MyDrive/mne_data/NeuroIPS_Hack_Sleep/AttnSleepTLnew1_checkpoints\"\n",
        "wandb_run.config.update({\"lr\":lr,\"batch_size\":batch_size,\"channel\":channel,\"save_path\":save_path})\n",
        "\n",
        "train_bacc = EpochScoring(\n",
        "    scoring='balanced_accuracy', on_train=True, name='train_bacc',\n",
        "    lower_is_better=False)\n",
        "cp = Checkpoint(monitor = 'train_bacc_best',\n",
        "                f_params = \"params_{last_epoch[epoch]}.pt\", \n",
        "                f_optimizer = \"optimizer_{last_epoch[epoch]}.pt\", \n",
        "                f_criterion = \"criterion_{last_epoch[epoch]}.pt\",\n",
        "                dirname = save_path)\n",
        "train_end_cp = TrainEndCheckpoint(\n",
        "                f_params = \"params_{last_epoch[epoch]}.pt\", \n",
        "                f_optimizer = \"optimizer_{last_epoch[epoch]}.pt\", \n",
        "                f_criterion = \"criterion_{last_epoch[epoch]}.pt\",\n",
        "                dirname=save_path)\n",
        "# f1, accuracy\n",
        "callbacks = [('train_bal_acc', train_bacc),\n",
        "             (\"checkpoint\",cp),\n",
        "             (\"train_end_cp\",train_end_cp),\n",
        "             (\"wandb\",WandbLogger(wandb_run,save_model=False))\n",
        "            ]\n",
        "\n",
        "clf5 = NNClassifier(\n",
        "  module = AttnSleep,\n",
        "  criterion = weighted_CrossEntropyLoss,\n",
        "  optimizer=torch.optim.Adam,\n",
        "  train_split=None,\n",
        "  iterator_train = DataLoader,\n",
        "  optimizer__lr=lr,\n",
        "  batch_size=batch_size,\n",
        "  callbacks=callbacks,\n",
        "  warm_start = True,\n",
        "  device=device\n",
        ")\n",
        "print(clf5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized](\n",
            "  module=<class '__main__.AttnSleep'>,\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BnS6GB2i5Tb7"
      },
      "source": [
        "clf5.initialize() # This is important!\n",
        "\n",
        "# Loading from the group 1 trained model\n",
        "load_path =\"/content/drive/MyDrive/mne_data/NeuroIPS_Hack_Sleep/AttnSleep2r_checkpoints\"\n",
        "clf5.load_params(\n",
        "    f_params=load_path+'/params_70.pt', f_optimizer=load_path+'/optimizer_70.pt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c0M3AsoS5Tb8",
        "outputId": "aa90d0d9-a2d0-4dfc-a9b0-65116601d457"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valA0.y\n",
        "y_pred = clf5.predict(valA0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Leaderboard Validation data\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py:652: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)\n",
            "  return torch.max_pool1d(input, kernel_size, stride, padding, dilation, ceil_mode)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Leaderboard Validation data\n",
            "[[5132  468   54    4    1  351]\n",
            " [ 199  543  509    9    9  403]\n",
            " [  85  335 3544  335  238  498]\n",
            " [   3    6  113  213  313   56]\n",
            " [   0    0   10   37  365    2]\n",
            " [  43  147  125    1    0 1291]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.85      0.89      6010\n",
            "           1       0.36      0.32      0.34      1672\n",
            "           2       0.81      0.70      0.75      5035\n",
            "           3       0.36      0.30      0.33       704\n",
            "           4       0.39      0.88      0.54       414\n",
            "           5       0.50      0.80      0.61      1607\n",
            "\n",
            "    accuracy                           0.72     15442\n",
            "   macro avg       0.56      0.65      0.58     15442\n",
            "weighted avg       0.75      0.72      0.72     15442\n",
            "\n",
            "0.6184415147452027\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LLz7m_inSC6G",
        "outputId": "c67eb25c-87b1-4825-9390-d76bcbc817a0"
      },
      "source": [
        "#checking if loaded properly\n",
        "# Untrained\n",
        "y_true = valC0.y\n",
        "y_pred = clf5.predict(valC0.X)\n",
        "\n",
        "print(\"From Untrained - Prediction on Phase 2 Validation Data before TL training\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Untrained - Prediction on Phase 2 Validation Data before TL training\n",
            "[[7090  469   48    5    5  192]\n",
            " [ 296  614  216   52   18  443]\n",
            " [ 185  642 1271  607  442 1557]\n",
            " [  18    2   18  147  471   33]\n",
            " [   3    0    0    7  134    1]\n",
            " [ 121  295  170   36   14  946]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.91      0.91      7809\n",
            "           1       0.30      0.37      0.34      1639\n",
            "           2       0.74      0.27      0.40      4704\n",
            "           3       0.17      0.21      0.19       689\n",
            "           4       0.12      0.92      0.22       145\n",
            "           5       0.30      0.60      0.40      1582\n",
            "\n",
            "    accuracy                           0.62     16568\n",
            "   macro avg       0.43      0.55      0.41     16568\n",
            "weighted avg       0.71      0.62      0.62     16568\n",
            "\n",
            "0.4648407938486785\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eqX-TkoO5Tb9",
        "outputId": "cb1d38b4-8827-410d-d22f-b16e1e01cb66"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valC0.y\n",
        "y_pred = clf5.predict(valC0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Phase 2 Validation Data before TL training\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Phase 2 Validation Data before TL training\n",
            "[[7620  113   48    4    0   24]\n",
            " [ 413  510  559    9    1  147]\n",
            " [  75  248 4011  120   19  231]\n",
            " [   1    1  258  329  100    0]\n",
            " [   0    0    7   44   94    0]\n",
            " [ 140  207  336   10    0  889]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.98      0.95      7809\n",
            "           1       0.47      0.31      0.38      1639\n",
            "           2       0.77      0.85      0.81      4704\n",
            "           3       0.64      0.48      0.55       689\n",
            "           4       0.44      0.65      0.52       145\n",
            "           5       0.69      0.56      0.62      1582\n",
            "\n",
            "    accuracy                           0.81     16568\n",
            "   macro avg       0.66      0.64      0.64     16568\n",
            "weighted avg       0.80      0.81      0.80     16568\n",
            "\n",
            "0.7153917831791735\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bv1dGZKI5Tb-",
        "outputId": "daaa7a89-6ada-4c2b-ab7d-038cf710ed2f"
      },
      "source": [
        "start = time.time()\n",
        "clf5.fit(valC0, y=None, epochs=n_epochs)\n",
        "end = time.time()\n",
        "print(f'time taken : {end-start}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  epoch    train_bacc    train_loss    cp     dur\n",
            "-------  ------------  ------------  ----  ------\n",
            "      1        \u001b[36m0.5604\u001b[0m        \u001b[32m0.7493\u001b[0m     +  3.5063\n",
            "      2        0.5598        \u001b[32m0.6494\u001b[0m        1.8288\n",
            "      3        \u001b[36m0.5960\u001b[0m        \u001b[32m0.6319\u001b[0m     +  1.8325\n",
            "      4        \u001b[36m0.6185\u001b[0m        \u001b[32m0.5626\u001b[0m     +  1.8338\n",
            "      5        \u001b[36m0.6557\u001b[0m        \u001b[32m0.5239\u001b[0m     +  1.9436\n",
            "      6        \u001b[36m0.6666\u001b[0m        \u001b[32m0.5075\u001b[0m     +  1.9028\n",
            "      7        \u001b[36m0.6869\u001b[0m        \u001b[32m0.5017\u001b[0m     +  1.8480\n",
            "      8        0.6829        \u001b[32m0.4941\u001b[0m        1.8449\n",
            "      9        \u001b[36m0.6921\u001b[0m        \u001b[32m0.4771\u001b[0m     +  1.9497\n",
            "     10        \u001b[36m0.7001\u001b[0m        \u001b[32m0.4696\u001b[0m     +  1.8465\n",
            "     11        \u001b[36m0.7022\u001b[0m        \u001b[32m0.4595\u001b[0m     +  1.8442\n",
            "     12        \u001b[36m0.7240\u001b[0m        \u001b[32m0.4471\u001b[0m     +  1.8710\n",
            "     13        \u001b[36m0.7259\u001b[0m        \u001b[32m0.4401\u001b[0m     +  1.9547\n",
            "     14        \u001b[36m0.7354\u001b[0m        \u001b[32m0.4373\u001b[0m     +  1.8433\n",
            "     15        \u001b[36m0.7399\u001b[0m        \u001b[32m0.4290\u001b[0m     +  1.8422\n",
            "     16        0.7334        \u001b[32m0.4226\u001b[0m        1.8441\n",
            "     17        \u001b[36m0.7572\u001b[0m        \u001b[32m0.4084\u001b[0m     +  1.9409\n",
            "     18        0.7418        0.4086        1.8627\n",
            "     19        \u001b[36m0.7619\u001b[0m        \u001b[32m0.4007\u001b[0m     +  1.8438\n",
            "     20        \u001b[36m0.7686\u001b[0m        \u001b[32m0.3864\u001b[0m     +  1.8378\n",
            "     21        0.7618        \u001b[32m0.3859\u001b[0m        1.9475\n",
            "     22        0.7683        \u001b[32m0.3768\u001b[0m        1.8411\n",
            "     23        \u001b[36m0.7780\u001b[0m        \u001b[32m0.3702\u001b[0m     +  1.8540\n",
            "     24        0.7775        \u001b[32m0.3600\u001b[0m        1.8757\n",
            "     25        \u001b[36m0.7817\u001b[0m        0.3677     +  1.9469\n",
            "     26        \u001b[36m0.7991\u001b[0m        \u001b[32m0.3524\u001b[0m     +  1.8496\n",
            "     27        \u001b[36m0.8072\u001b[0m        \u001b[32m0.3370\u001b[0m     +  1.8531\n",
            "     28        0.7867        0.3437        1.8463\n",
            "     29        \u001b[36m0.8077\u001b[0m        \u001b[32m0.3256\u001b[0m     +  1.9477\n",
            "     30        \u001b[36m0.8134\u001b[0m        \u001b[32m0.3202\u001b[0m     +  1.8824\n",
            "     31        \u001b[36m0.8262\u001b[0m        \u001b[32m0.3126\u001b[0m     +  1.8410\n",
            "     32        \u001b[36m0.8435\u001b[0m        \u001b[32m0.3015\u001b[0m     +  1.8445\n",
            "     33        0.8145        0.3146        1.9578\n",
            "     34        0.8216        0.3074        1.8411\n",
            "     35        0.8307        \u001b[32m0.2953\u001b[0m        1.8370\n",
            "     36        0.8408        \u001b[32m0.2860\u001b[0m        1.8622\n",
            "     37        \u001b[36m0.8461\u001b[0m        \u001b[32m0.2835\u001b[0m     +  1.9446\n",
            "     38        0.8410        \u001b[32m0.2753\u001b[0m        1.8355\n",
            "     39        0.8317        0.2761        1.8374\n",
            "     40        \u001b[36m0.8513\u001b[0m        \u001b[32m0.2575\u001b[0m     +  1.8417\n",
            "     41        0.8495        \u001b[32m0.2511\u001b[0m        1.9461\n",
            "     42        \u001b[36m0.8607\u001b[0m        \u001b[32m0.2408\u001b[0m     +  1.8631\n",
            "     43        \u001b[36m0.8691\u001b[0m        \u001b[32m0.2406\u001b[0m     +  1.8372\n",
            "     44        0.8639        \u001b[32m0.2339\u001b[0m        1.8402\n",
            "     45        \u001b[36m0.8747\u001b[0m        \u001b[32m0.2248\u001b[0m     +  1.8407\n",
            "     46        \u001b[36m0.8759\u001b[0m        \u001b[32m0.2217\u001b[0m     +  1.9493\n",
            "     47        \u001b[36m0.8821\u001b[0m        0.2235     +  1.8371\n",
            "     48        0.8816        \u001b[32m0.2158\u001b[0m        1.8790\n",
            "     49        0.8732        0.2221        1.8378\n",
            "     50        0.8799        0.2190        1.8400\n",
            "     51        0.8731        0.2189        1.9516\n",
            "     52        \u001b[36m0.8885\u001b[0m        \u001b[32m0.2044\u001b[0m     +  1.8355\n",
            "     53        \u001b[36m0.9007\u001b[0m        \u001b[32m0.1876\u001b[0m     +  1.8689\n",
            "     54        0.8910        0.1900        1.8448\n",
            "     55        0.8960        \u001b[32m0.1825\u001b[0m        1.8344\n",
            "     56        \u001b[36m0.9024\u001b[0m        \u001b[32m0.1795\u001b[0m     +  1.9463\n",
            "     57        0.8930        \u001b[32m0.1750\u001b[0m        1.8357\n",
            "     58        \u001b[36m0.9085\u001b[0m        \u001b[32m0.1724\u001b[0m     +  1.8425\n",
            "     59        0.9076        \u001b[32m0.1616\u001b[0m        1.8698\n",
            "     60        0.9020        \u001b[32m0.1581\u001b[0m        1.8369\n",
            "     61        \u001b[36m0.9107\u001b[0m        \u001b[32m0.1445\u001b[0m     +  1.9452\n",
            "     62        \u001b[36m0.9268\u001b[0m        \u001b[32m0.1387\u001b[0m     +  1.8402\n",
            "     63        \u001b[36m0.9371\u001b[0m        \u001b[32m0.1280\u001b[0m     +  1.8392\n",
            "     64        0.9259        0.1344        1.8378\n",
            "     65        0.9232        0.1360        1.8675\n",
            "     66        0.9217        0.1395        1.9505\n",
            "     67        0.9297        0.1353        1.8411\n",
            "     68        0.9306        \u001b[32m0.1268\u001b[0m        1.8382\n",
            "     69        0.9294        \u001b[32m0.1191\u001b[0m        1.8411\n",
            "     70        \u001b[36m0.9447\u001b[0m        \u001b[32m0.1180\u001b[0m     +  1.8437\n",
            "     71        0.9383        \u001b[32m0.1161\u001b[0m        1.9743\n",
            "     72        \u001b[36m0.9448\u001b[0m        \u001b[32m0.1136\u001b[0m     +  1.8416\n",
            "     73        0.9320        0.1212        1.8443\n",
            "     74        0.9298        0.1178        1.8431\n",
            "     75        0.9224        0.1240        1.8388\n",
            "     76        0.9345        0.1247        1.9424\n",
            "     77        0.9398        0.1179        1.8617\n",
            "     78        0.9393        0.1250        1.8371\n",
            "     79        0.9384        0.1216        1.8394\n",
            "     80        0.9346        0.1218        1.8384\n",
            "     81        0.9415        0.1261        1.8385\n",
            "     82        0.9423        \u001b[32m0.1128\u001b[0m        1.9495\n",
            "     83        0.9347        0.1281        1.8643\n",
            "     84        0.9399        \u001b[32m0.1092\u001b[0m        1.8359\n",
            "     85        0.9437        0.1115        1.8359\n",
            "     86        \u001b[36m0.9499\u001b[0m        \u001b[32m0.1013\u001b[0m     +  1.8387\n",
            "     87        \u001b[36m0.9527\u001b[0m        \u001b[32m0.0941\u001b[0m     +  1.9473\n",
            "     88        0.9484        \u001b[32m0.0914\u001b[0m        1.8372\n",
            "     89        \u001b[36m0.9562\u001b[0m        \u001b[32m0.0814\u001b[0m     +  1.8711\n",
            "     90        0.9524        \u001b[32m0.0803\u001b[0m        1.8364\n",
            "     91        \u001b[36m0.9624\u001b[0m        \u001b[32m0.0720\u001b[0m     +  1.8390\n",
            "     92        \u001b[36m0.9696\u001b[0m        \u001b[32m0.0679\u001b[0m     +  1.8455\n",
            "     93        0.9669        0.0695        1.9476\n",
            "     94        0.9684        \u001b[32m0.0608\u001b[0m        1.8339\n",
            "     95        0.9691        \u001b[32m0.0551\u001b[0m        1.8597\n",
            "     96        0.9635        0.0642        1.8407\n",
            "     97        \u001b[36m0.9712\u001b[0m        0.0591     +  1.8430\n",
            "     98        0.9694        0.0574        1.9462\n",
            "     99        \u001b[36m0.9733\u001b[0m        0.0575     +  1.8362\n",
            "    100        0.9691        0.0597        1.8706\n",
            "time taken : 193.40388917922974\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "22qbuExW5Tb_",
        "outputId": "eb7c20a3-b6c5-4ae5-ee2d-0e9b71de0cf9"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valC0.y\n",
        "y_pred = clf5.predict(valC0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Phase 2 Validation data after TL training on Phase 2 val\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Phase 2 Validation data after TL training on Phase 2 val\n",
            "[[7773   19    6    0    0   11]\n",
            " [  26 1517   76    6    0   14]\n",
            " [   6   16 4635    7    0   40]\n",
            " [   0    0   67  622    0    0]\n",
            " [   0    0    1    1  143    0]\n",
            " [   7    4   38    1    0 1532]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      1.00      1.00      7809\n",
            "           1       0.97      0.93      0.95      1639\n",
            "           2       0.96      0.99      0.97      4704\n",
            "           3       0.98      0.90      0.94       689\n",
            "           4       1.00      0.99      0.99       145\n",
            "           5       0.96      0.97      0.96      1582\n",
            "\n",
            "    accuracy                           0.98     16568\n",
            "   macro avg       0.98      0.96      0.97     16568\n",
            "weighted avg       0.98      0.98      0.98     16568\n",
            "\n",
            "0.9690586799902011\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J5gslKOo5TcA",
        "outputId": "b2a5bf2b-51d4-40f5-e10a-ff8c17bf95e1"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valA0.y\n",
        "y_pred = clf5.predict(valA0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Leaderboard Validation data after TL training on Phase 2 val\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Leaderboard Validation data after TL training on Phase 2 val\n",
            "[[5286  325  153    6    4  236]\n",
            " [ 297  473  603    0    0  299]\n",
            " [ 159  413 4216   13    1  233]\n",
            " [   0    5  639   53    5    2]\n",
            " [   0    0  264  105   45    0]\n",
            " [ 181  267  231    0    0  928]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.89      0.88      0.89      6010\n",
            "           1       0.32      0.28      0.30      1672\n",
            "           2       0.69      0.84      0.76      5035\n",
            "           3       0.30      0.08      0.12       704\n",
            "           4       0.82      0.11      0.19       414\n",
            "           5       0.55      0.58      0.56      1607\n",
            "\n",
            "    accuracy                           0.71     15442\n",
            "   macro avg       0.59      0.46      0.47     15442\n",
            "weighted avg       0.70      0.71      0.69     15442\n",
            "\n",
            "0.5887597621244347\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dV36xw8A5TcA",
        "outputId": "410280cf-c1aa-4d5b-ea6e-5b8a5f332528"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = trainA0.y\n",
        "y_pred = clf5.predict(trainA0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on original Group 1 training data after TL training on Phase 2 Val data\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on original Group 1 training data after TL training on Phase 2 Val data\n",
            "[[21398  1036   544    30     5  1030]\n",
            " [ 1252  1726  2869     3     1  2090]\n",
            " [  432  1516 32128   215     7  1685]\n",
            " [   16    11  4239   855   103    23]\n",
            " [    2     0  1598   904   552     1]\n",
            " [  817  1067  3330    21     2  9037]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.89      0.89      0.89     24043\n",
            "           1       0.32      0.22      0.26      7941\n",
            "           2       0.72      0.89      0.80     35983\n",
            "           3       0.42      0.16      0.24      5247\n",
            "           4       0.82      0.18      0.30      3057\n",
            "           5       0.65      0.63      0.64     14274\n",
            "\n",
            "    accuracy                           0.73     90545\n",
            "   macro avg       0.64      0.50      0.52     90545\n",
            "weighted avg       0.71      0.73      0.70     90545\n",
            "\n",
            "0.6094841107515762\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "siB-y0QV5TcC",
        "outputId": "208bfaac-2b2b-4cac-e954-03d5840b0136"
      },
      "source": [
        "predict_leaderboard_unlabelled(clf=clf5,save_fname=\"AttnSleep_TL_slightTL\", x_test_data = testC.X[:,channel,:].unsqueeze(1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([25756, 1, 3000])\n",
            "Checking if all classes have been predicted\n",
            "[0 1 2 3 4 5]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k5YcCFo95TcD"
      },
      "source": [
        "This model got 63.8% on the Sleep Leaderboard. Without the TL it was at 60%.. Not such a great improvement.. Seems like Overfitting on the 5 subject data wasn't enough to generalize to all the leaderboard data.."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443,
          "referenced_widgets": [
            "4d1a67dc139d4a149c6d6d36a03ba304",
            "13b93d22ef074df4aa4138e0761ed3a2",
            "9fc04a8d2d6145f68491b83e25a413c3",
            "b2330e6f3071449ca0bb228cf26dcf7f",
            "775c17a367f846579bbf4ec0a1ebc62a",
            "d665e0d4a0854cd2b03e2f0755dac467",
            "7e722ec324754d769cfa985d5820efcf",
            "fec6b8b9b0e44abca1d10c8c6f654bb5"
          ]
        },
        "id": "omSnOzU55TcD",
        "outputId": "540911ff-1da7-44d0-c11d-f45b4c0aee62"
      },
      "source": [
        "wandb.finish()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<br/>Waiting for W&B process to finish, PID 634<br/>Program ended successfully."
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4d1a67dc139d4a149c6d6d36a03ba304",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "VBox(children=(Label(value=' 0.00MB of 0.00MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Find user logs for this run at: <code>/content/wandb/run-20210927_153747-2sq42dwg/logs/debug.log</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Find internal logs for this run at: <code>/content/wandb/run-20210927_153747-2sq42dwg/logs/debug-internal.log</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<h3>Run summary:</h3><br/><style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    </style><table class=\"wandb\">\n",
              "<tr><td>dur</td><td>1.87056</td></tr><tr><td>train_bacc</td><td>0.96911</td></tr><tr><td>train_loss</td><td>0.05969</td></tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<h3>Run history:</h3><br/><style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    </style><table class=\"wandb\">\n",
              "<tr><td>dur</td><td>█▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>train_bacc</td><td>▁▂▃▃▃▄▄▄▄▅▅▅▆▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇█▇▇▇█▇██████</td></tr><tr><td>train_loss</td><td>█▇▆▅▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁</td></tr></table><br/>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                    <br/>Synced <strong style=\"color:#cdcd00\">AttnSleep_TL_new1</strong>: <a href=\"https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/2sq42dwg\" target=\"_blank\">https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/2sq42dwg</a><br/>\n",
              "                "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zDtVVwRYs_cS"
      },
      "source": [
        "# Other Tries"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wJ-wLZbrtNHW"
      },
      "source": [
        "The following code are not necessary for replicating the model and performance submitted to the Hackathon but show the other techniques tried"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HBq3XfTAWU0q"
      },
      "source": [
        "# 2nd Level TL using model on which leaderboard TL has been done and prediction on final test set - *Not* the model whose predictions was submitted"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XmdE98YCbQZJ"
      },
      "source": [
        "The leaderboard data can be used in testing phase\n",
        "\n",
        "I'm not sure if the target data provided for the test is the same as the labelled target in the leaderboard"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NB-N7CIqe5MF",
        "outputId": "ed71509a-fe0f-4a8d-8e81-bde1f8507a7b"
      },
      "source": [
        "valC = get_valC()\n",
        "testC = get_testC()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "phase 2 labelled load time = 0.3309566259384155 min\n",
            "There are 16568 trials with 2 electrodes and 3000 time samples\n",
            "(16568, 2, 3000) (16568,)\n",
            "[7809 1639 4704  689  145 1582]\n",
            "Z-scoring time 7.114246129989624 sec\n",
            "Torch dataset time 0.0020368099212646484 sec = 3.394683202107747e-05 min\n",
            "phase 2 final test set load time = 0.3870764295260111 min\n",
            "There are 25756 trials with 2 electrodes and 3000 time samples\n",
            "Z-scoring time 1.73142409324646 sec\n",
            "Torch dataset time 6.651878356933594e-05 sec = 1.1086463928222657e-06 min\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Rgjb-ktexXQ"
      },
      "source": [
        "valC0 = Dataset(valC.X[:,0,:].unsqueeze(1),valC.y.long())\n",
        "testC0 = Dataset(testC.X[:,0,:].unsqueeze(1),testC.y.long())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OBH0FAqWWZGe"
      },
      "source": [
        "# Load the leaderboard overfitted model that gave maximum performance till now"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 179
        },
        "id": "a5h7X5zyi-zs",
        "outputId": "026eba4f-cf52-44c7-f810-1ac48ada1a3b"
      },
      "source": [
        "# Load the Channel 0, batch_size 1024 model\n",
        "# starting wandb init\n",
        "wandb_run = wandb.init(name = \"AttnSleep_TL_new\", project='cnn-fulltrain', entity='sleep_hacking')\n",
        "\n",
        "# model = clf.module\n",
        "\n",
        "# batch_size\n",
        "lr = 1e-3 #change\n",
        "batch_size = 1024\n",
        "n_epochs = 100\n",
        "\n",
        "channel = 0\n",
        "save_path = \"/content/drive/MyDrive/mne_data/NeuroIPS_Hack_Sleep/AttnSleepTLnew0_checkpoints\"\n",
        "wandb_run.config.update({\"lr\":lr,\"batch_size\":batch_size,\"channel\":channel,\"save_path\":save_path})\n",
        "\n",
        "train_bacc = EpochScoring(\n",
        "    scoring='balanced_accuracy', on_train=True, name='train_bacc',\n",
        "    lower_is_better=False)\n",
        "cp = Checkpoint(monitor = 'train_bacc_best',\n",
        "                f_params = \"params_{last_epoch[epoch]}.pt\", \n",
        "                f_optimizer = \"optimizer_{last_epoch[epoch]}.pt\", \n",
        "                f_criterion = \"criterion_{last_epoch[epoch]}.pt\",\n",
        "                dirname = save_path)\n",
        "train_end_cp = TrainEndCheckpoint(\n",
        "                f_params = \"params_{last_epoch[epoch]}.pt\", \n",
        "                f_optimizer = \"optimizer_{last_epoch[epoch]}.pt\", \n",
        "                f_criterion = \"criterion_{last_epoch[epoch]}.pt\",\n",
        "                dirname=save_path)\n",
        "\n",
        "callbacks = [('train_bal_acc', train_bacc),\n",
        "             (\"checkpoint\",cp),\n",
        "             (\"train_end_cp\",train_end_cp),\n",
        "             (\"wandb\",WandbLogger(wandb_run,save_model=False))\n",
        "            ]\n",
        "\n",
        "clf5 = NNClassifier(\n",
        "  module = AttnSleep,\n",
        "  criterion = weighted_CrossEntropyLoss,\n",
        "  optimizer=torch.optim.Adam,\n",
        "  train_split=None,\n",
        "  iterator_train = DataLoader,\n",
        "  optimizer__lr=lr,\n",
        "  batch_size=batch_size,\n",
        "  callbacks=callbacks,\n",
        "  warm_start = True,\n",
        "  device=device\n",
        ")\n",
        "print(clf5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.12.2<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">AttnSleep_TL_new</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/sleep_hacking/cnn-fulltrain\" target=\"_blank\">https://wandb.ai/sleep_hacking/cnn-fulltrain</a><br/>\n",
              "                Run page: <a href=\"https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/1e0wpixp\" target=\"_blank\">https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/1e0wpixp</a><br/>\n",
              "                Run data is saved locally in <code>/content/wandb/run-20210927_143637-1e0wpixp</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized](\n",
            "  module=<class '__main__.AttnSleep'>,\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CiJVFBe4i-zw"
      },
      "source": [
        "clf5.initialize() # This is important!\n",
        "load_path =\"/content/drive/MyDrive/mne_data/NeuroIPS_Hack_Sleep/AttnSleepTL0_checkpoints\"\n",
        "\n",
        "clf5.load_params(\n",
        "    f_params=load_path+'/params_100.pt', f_optimizer=load_path+'/optimizer_100.pt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ScNg-nlIi-zw",
        "outputId": "8d8c9b69-ee74-4e6e-a0cf-319a75b91641"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valA0.y\n",
        "y_pred = clf5.predict(valA0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Leaderboard Validation data\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Leaderboard Validation data\n",
            "[[5848   55   23    1    2   81]\n",
            " [  25 1474  119    6    6   42]\n",
            " [  23   43 4657  197   72   43]\n",
            " [   0    0   15  638   50    1]\n",
            " [   0    0    1    1  412    0]\n",
            " [  10   15   27    0    0 1555]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      0.97      0.98      6010\n",
            "           1       0.93      0.88      0.90      1672\n",
            "           2       0.96      0.92      0.94      5035\n",
            "           3       0.76      0.91      0.82       704\n",
            "           4       0.76      1.00      0.86       414\n",
            "           5       0.90      0.97      0.93      1607\n",
            "\n",
            "    accuracy                           0.94     15442\n",
            "   macro avg       0.88      0.94      0.91     15442\n",
            "weighted avg       0.95      0.94      0.95     15442\n",
            "\n",
            "0.923122533570641\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oYE6ZGupsGwl",
        "outputId": "c32ba25d-c1b4-4dca-ac99-a3084b23bc81"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valC0.y\n",
        "y_pred = clf5.predict(valC0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Phase 2 Validation Data before TL training\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Phase 2 Validation Data before TL training\n",
            "[[7235  341   56    9    3  165]\n",
            " [ 319  412  501   67   12  328]\n",
            " [ 117  323 2715  915  353  281]\n",
            " [   5    3   31  198  450    2]\n",
            " [   2    0    0    6  137    0]\n",
            " [ 107  234  400   57   13  771]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.93      0.93      7809\n",
            "           1       0.31      0.25      0.28      1639\n",
            "           2       0.73      0.58      0.65      4704\n",
            "           3       0.16      0.29      0.20       689\n",
            "           4       0.14      0.94      0.25       145\n",
            "           5       0.50      0.49      0.49      1582\n",
            "\n",
            "    accuracy                           0.69     16568\n",
            "   macro avg       0.46      0.58      0.47     16568\n",
            "weighted avg       0.73      0.69      0.71     16568\n",
            "\n",
            "0.5568759416152871\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cCyfTT11i-z0",
        "outputId": "863dd0b9-d0b7-49ee-cbd8-5ce75b9e9a91"
      },
      "source": [
        "start = time.time()\n",
        "clf5.fit(valC0, y=None, epochs=n_epochs)\n",
        "end = time.time()\n",
        "print(f'time taken : {end-start}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  epoch    train_bacc    train_loss    cp     dur\n",
            "-------  ------------  ------------  ----  ------\n",
            "      1        \u001b[36m0.5004\u001b[0m        \u001b[32m1.2982\u001b[0m     +  2.3097\n",
            "      2        \u001b[36m0.5097\u001b[0m        \u001b[32m0.8621\u001b[0m     +  1.8406\n",
            "      3        \u001b[36m0.5890\u001b[0m        \u001b[32m0.6924\u001b[0m     +  1.9452\n",
            "      4        0.5725        \u001b[32m0.6719\u001b[0m        1.8371\n",
            "      5        \u001b[36m0.6155\u001b[0m        \u001b[32m0.6118\u001b[0m     +  1.8451\n",
            "      6        \u001b[36m0.6579\u001b[0m        \u001b[32m0.5650\u001b[0m     +  1.8641\n",
            "      7        \u001b[36m0.6687\u001b[0m        \u001b[32m0.5269\u001b[0m     +  1.9545\n",
            "      8        \u001b[36m0.6736\u001b[0m        \u001b[32m0.5111\u001b[0m     +  1.8410\n",
            "      9        \u001b[36m0.6981\u001b[0m        \u001b[32m0.4863\u001b[0m     +  1.8510\n",
            "     10        \u001b[36m0.7148\u001b[0m        \u001b[32m0.4744\u001b[0m     +  1.8334\n",
            "     11        0.7032        \u001b[32m0.4693\u001b[0m        1.8448\n",
            "     12        \u001b[36m0.7222\u001b[0m        \u001b[32m0.4613\u001b[0m     +  1.9981\n",
            "     13        \u001b[36m0.7257\u001b[0m        \u001b[32m0.4486\u001b[0m     +  1.8425\n",
            "     14        \u001b[36m0.7500\u001b[0m        \u001b[32m0.4334\u001b[0m     +  1.8387\n",
            "     15        0.7425        \u001b[32m0.4306\u001b[0m        1.8436\n",
            "     16        \u001b[36m0.7503\u001b[0m        \u001b[32m0.4230\u001b[0m     +  1.9559\n",
            "     17        0.7484        \u001b[32m0.4131\u001b[0m        1.8315\n",
            "     18        \u001b[36m0.7663\u001b[0m        \u001b[32m0.4023\u001b[0m     +  1.8753\n",
            "     19        \u001b[36m0.7804\u001b[0m        \u001b[32m0.3895\u001b[0m     +  1.8382\n",
            "     20        0.7762        \u001b[32m0.3866\u001b[0m        1.8328\n",
            "     21        \u001b[36m0.7866\u001b[0m        \u001b[32m0.3787\u001b[0m     +  1.9450\n",
            "     22        0.7781        \u001b[32m0.3723\u001b[0m        1.8381\n",
            "     23        \u001b[36m0.7910\u001b[0m        \u001b[32m0.3697\u001b[0m     +  1.8325\n",
            "     24        \u001b[36m0.7976\u001b[0m        \u001b[32m0.3585\u001b[0m     +  1.8711\n",
            "     25        \u001b[36m0.8049\u001b[0m        \u001b[32m0.3475\u001b[0m     +  1.8395\n",
            "     26        0.8006        \u001b[32m0.3410\u001b[0m        1.9607\n",
            "     27        0.8003        0.3426        1.8390\n",
            "     28        \u001b[36m0.8256\u001b[0m        \u001b[32m0.3334\u001b[0m     +  1.8370\n",
            "     29        0.7888        0.3506        1.8417\n",
            "     30        0.7786        0.3438        1.9780\n",
            "     31        0.8084        \u001b[32m0.3246\u001b[0m        1.8334\n",
            "     32        0.8101        0.3256        1.8368\n",
            "     33        \u001b[36m0.8266\u001b[0m        \u001b[32m0.3046\u001b[0m     +  1.8468\n",
            "     34        \u001b[36m0.8465\u001b[0m        \u001b[32m0.2881\u001b[0m     +  1.9839\n",
            "     35        0.8412        \u001b[32m0.2838\u001b[0m        1.8427\n",
            "     36        \u001b[36m0.8476\u001b[0m        \u001b[32m0.2770\u001b[0m     +  1.8710\n",
            "     37        \u001b[36m0.8588\u001b[0m        \u001b[32m0.2562\u001b[0m     +  1.8433\n",
            "     38        0.8570        0.2565        1.8372\n",
            "     39        \u001b[36m0.8714\u001b[0m        \u001b[32m0.2467\u001b[0m     +  1.9590\n",
            "     40        0.8681        \u001b[32m0.2386\u001b[0m        1.8403\n",
            "     41        \u001b[36m0.8738\u001b[0m        \u001b[32m0.2345\u001b[0m     +  1.8485\n",
            "     42        \u001b[36m0.8762\u001b[0m        0.2397     +  1.8645\n",
            "     43        0.8678        0.2402        1.8497\n",
            "     44        0.8501        0.2490        1.8452\n",
            "     45        0.8410        0.2547        1.9471\n",
            "     46        0.8646        \u001b[32m0.2291\u001b[0m        1.8401\n",
            "     47        0.8686        0.2302        1.8449\n",
            "     48        \u001b[36m0.8891\u001b[0m        \u001b[32m0.2123\u001b[0m     +  1.8672\n",
            "     49        \u001b[36m0.8980\u001b[0m        \u001b[32m0.1937\u001b[0m     +  1.8372\n",
            "     50        0.8905        0.2022        1.8373\n",
            "     51        \u001b[36m0.8982\u001b[0m        0.1954     +  1.8353\n",
            "     52        0.8932        0.1995        1.9573\n",
            "     53        0.8960        \u001b[32m0.1900\u001b[0m        1.8615\n",
            "     54        \u001b[36m0.9019\u001b[0m        \u001b[32m0.1827\u001b[0m     +  1.8342\n",
            "     55        0.9002        \u001b[32m0.1767\u001b[0m        1.8353\n",
            "     56        \u001b[36m0.9124\u001b[0m        \u001b[32m0.1644\u001b[0m     +  1.9614\n",
            "     57        0.9015        \u001b[32m0.1632\u001b[0m        1.8372\n",
            "     58        0.9026        0.1749        1.8392\n",
            "     59        0.9118        \u001b[32m0.1594\u001b[0m        1.8654\n",
            "     60        \u001b[36m0.9194\u001b[0m        \u001b[32m0.1516\u001b[0m     +  1.8433\n",
            "     61        0.9133        0.1551        1.8475\n",
            "     62        \u001b[36m0.9197\u001b[0m        \u001b[32m0.1433\u001b[0m     +  1.8352\n",
            "     63        \u001b[36m0.9246\u001b[0m        \u001b[32m0.1358\u001b[0m     +  1.9538\n",
            "     64        \u001b[36m0.9350\u001b[0m        \u001b[32m0.1265\u001b[0m     +  1.8436\n",
            "     65        \u001b[36m0.9354\u001b[0m        0.1276     +  1.8628\n",
            "     66        \u001b[36m0.9380\u001b[0m        \u001b[32m0.1204\u001b[0m     +  1.8464\n",
            "     67        \u001b[36m0.9380\u001b[0m        0.1242     +  1.8453\n",
            "     68        \u001b[36m0.9449\u001b[0m        \u001b[32m0.1045\u001b[0m     +  1.8429\n",
            "     69        0.9447        0.1090        1.9483\n",
            "     70        0.9406        0.1104        1.8267\n",
            "     71        \u001b[36m0.9510\u001b[0m        \u001b[32m0.1026\u001b[0m     +  1.8604\n",
            "     72        0.9459        0.1085        1.8361\n",
            "     73        0.9455        0.1058        1.8371\n",
            "     74        0.9451        0.1033        1.8381\n",
            "     75        0.9422        \u001b[32m0.1012\u001b[0m        1.9458\n",
            "     76        0.9495        0.1043        1.8317\n",
            "     77        0.9435        0.1076        1.8577\n",
            "     78        0.9422        0.1129        1.8404\n",
            "     79        0.9408        0.1103        1.8298\n",
            "     80        \u001b[36m0.9517\u001b[0m        \u001b[32m0.0999\u001b[0m     +  1.8539\n",
            "     81        \u001b[36m0.9528\u001b[0m        \u001b[32m0.0976\u001b[0m     +  1.8620\n",
            "     82        0.9523        \u001b[32m0.0923\u001b[0m        1.9492\n",
            "     83        0.9435        \u001b[32m0.0915\u001b[0m        1.8655\n",
            "     84        \u001b[36m0.9534\u001b[0m        0.0917     +  1.8362\n",
            "     85        \u001b[36m0.9542\u001b[0m        \u001b[32m0.0884\u001b[0m     +  1.8357\n",
            "     86        0.9499        0.0947        1.8417\n",
            "     87        0.9525        0.0896        1.8403\n",
            "     88        \u001b[36m0.9554\u001b[0m        \u001b[32m0.0848\u001b[0m     +  1.9605\n",
            "     89        0.9466        0.0889        1.8730\n",
            "     90        0.9533        0.0894        1.8448\n",
            "     91        0.9514        0.0959        1.8387\n",
            "     92        0.9456        0.0905        1.8365\n",
            "     93        0.9544        0.0901        1.8387\n",
            "     94        0.9501        0.0943        1.8365\n",
            "     95        \u001b[36m0.9631\u001b[0m        0.0883     +  1.9931\n",
            "     96        0.9572        \u001b[32m0.0829\u001b[0m        1.8389\n",
            "     97        0.9572        \u001b[32m0.0800\u001b[0m        1.8419\n",
            "     98        0.9589        \u001b[32m0.0775\u001b[0m        1.8429\n",
            "     99        0.9609        \u001b[32m0.0746\u001b[0m        1.8369\n",
            "    100        \u001b[36m0.9649\u001b[0m        \u001b[32m0.0715\u001b[0m     +  1.8634\n",
            "time taken : 191.9069094657898\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vO3_a92tsq7L",
        "outputId": "587553e9-c2dd-4253-95be-f1ef0ccbc916"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valC0.y\n",
        "y_pred = clf5.predict(valC0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Phase 2 Validation data after TL training on Phase 2 val\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Phase 2 Validation data after TL training on Phase 2 val\n",
            "[[7790    7    3    1    1    7]\n",
            " [  39 1476   42    0    0   82]\n",
            " [   9   20 4415   21    1  238]\n",
            " [   0    0   42  647    0    0]\n",
            " [   0    0    0    1  144    0]\n",
            " [   9    0    5    0    0 1568]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      1.00      1.00      7809\n",
            "           1       0.98      0.90      0.94      1639\n",
            "           2       0.98      0.94      0.96      4704\n",
            "           3       0.97      0.94      0.95       689\n",
            "           4       0.99      0.99      0.99       145\n",
            "           5       0.83      0.99      0.90      1582\n",
            "\n",
            "    accuracy                           0.97     16568\n",
            "   macro avg       0.96      0.96      0.96     16568\n",
            "weighted avg       0.97      0.97      0.97     16568\n",
            "\n",
            "0.9529875599569695\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RVe8pOGAi-z0",
        "outputId": "6d948815-492f-4b56-be6c-6261010e08dc"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valA0.y\n",
        "y_pred = clf5.predict(valA0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Leaderboard Validation data after TL training on Phase 2 val\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Leaderboard Validation data after TL training on Phase 2 val\n",
            "[[5451  201   97   10    2  249]\n",
            " [ 249  469  453    0    2  499]\n",
            " [ 103  253 4124   20    2  533]\n",
            " [   1    2  612   83    2    4]\n",
            " [   0    0  236  140   38    0]\n",
            " [ 140   65  113    0    2 1287]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.91      0.91      6010\n",
            "           1       0.47      0.28      0.35      1672\n",
            "           2       0.73      0.82      0.77      5035\n",
            "           3       0.33      0.12      0.17       704\n",
            "           4       0.79      0.09      0.16       414\n",
            "           5       0.50      0.80      0.62      1607\n",
            "\n",
            "    accuracy                           0.74     15442\n",
            "   macro avg       0.62      0.50      0.50     15442\n",
            "weighted avg       0.74      0.74      0.72     15442\n",
            "\n",
            "0.6340654649550557\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hiLo5qZvi-z0",
        "outputId": "653c139f-6988-4923-810b-e8bbb124e5d0"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = trainA0.y\n",
        "y_pred = clf5.predict(trainA0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on original Group 1 training data after TL training on Phase 2 Val data\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on original Group 1 training data after TL training on Phase 2 Val data\n",
            "[[21626   662   436    26    12  1281]\n",
            " [ 1440   991  2215     7     0  3288]\n",
            " [  850  1001 30391   255     7  3479]\n",
            " [   26    16  4133   974    81    17]\n",
            " [    0     1  1372  1318   365     1]\n",
            " [ 1244   440  1988    30     2 10570]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.86      0.90      0.88     24043\n",
            "           1       0.32      0.12      0.18      7941\n",
            "           2       0.75      0.84      0.79     35983\n",
            "           3       0.37      0.19      0.25      5247\n",
            "           4       0.78      0.12      0.21      3057\n",
            "           5       0.57      0.74      0.64     14274\n",
            "\n",
            "    accuracy                           0.72     90545\n",
            "   macro avg       0.61      0.49      0.49     90545\n",
            "weighted avg       0.69      0.72      0.69     90545\n",
            "\n",
            "0.6018683529800133\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HonSkFv6uiTx"
      },
      "source": [
        "Good Example of Catastrophic Forgetting, but group staying at around 60% is interesting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3pPnwUaWi-z1",
        "outputId": "016e7513-3e3c-4589-c586-23f717cf516e"
      },
      "source": [
        "predict_leaderboard_unlabelled(clf=clf5,save_fname=\"AttnSleep_TL_final0\", x_test_data = testC.X[:,channel,:].unsqueeze(1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([25756, 1, 3000])\n",
            "Checking if all classes have been predicted\n",
            "[0 1 2 3 4 5]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cmxAPiwbi-z1"
      },
      "source": [
        "This model got 63.8% on the Sleep Leaderboard. Without the TL it was at 60%.. Not such a great improvement.. Seems like Overfitting on the 5 subject data wasn't enough to generalize to all the leaderboard data.."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443,
          "referenced_widgets": [
            "bcc1bf44d476410fb619ffdca6fc31ec",
            "1c2e25dd1fe543eb8b5b0f05ca83f627",
            "80ed1c51d00142e1a7bd2276b8f1228b",
            "ee7d3c67db304143966b2a7a4f4a258c",
            "5fe67647e3d94150a771abdbbedb855c",
            "d3ca02fb63c0483db6fd0f9bd8166b32",
            "ed99830578fc456d88207a3d1e4fc531",
            "4b80d35aba90474592c5e389fc78c471"
          ]
        },
        "id": "jIbsuadbi-z2",
        "outputId": "818164fc-5501-4481-dcfc-b5e86e2d520c"
      },
      "source": [
        "wandb.finish()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<br/>Waiting for W&B process to finish, PID 4460<br/>Program ended successfully."
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bcc1bf44d476410fb619ffdca6fc31ec",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "VBox(children=(Label(value=' 0.00MB of 0.00MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Find user logs for this run at: <code>/content/wandb/run-20210927_143637-1e0wpixp/logs/debug.log</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Find internal logs for this run at: <code>/content/wandb/run-20210927_143637-1e0wpixp/logs/debug-internal.log</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<h3>Run summary:</h3><br/><style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    </style><table class=\"wandb\">\n",
              "<tr><td>dur</td><td>1.86337</td></tr><tr><td>train_bacc</td><td>0.96495</td></tr><tr><td>train_loss</td><td>0.07148</td></tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<h3>Run history:</h3><br/><style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    </style><table class=\"wandb\">\n",
              "<tr><td>dur</td><td>█▃▂▁▁▁▃▂▃▁▃▁▁▃▂▃▁▁▁▁▁▁▃▂▁▁▁▃▁▁▁▁▃▁▁▂▁▁▁▁</td></tr><tr><td>train_bacc</td><td>▁▂▃▄▄▄▅▅▅▅▆▆▆▆▆▇▇▆▆▇▇▇▇▇▇███████████████</td></tr><tr><td>train_loss</td><td>█▅▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr></table><br/>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                    <br/>Synced <strong style=\"color:#cdcd00\">AttnSleep_TL_new</strong>: <a href=\"https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/1e0wpixp\" target=\"_blank\">https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/1e0wpixp</a><br/>\n",
              "                "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W5eVyQaCqfAo"
      },
      "source": [
        "# TL using first group trained model and target data of the final set (Primarily led to overfitting and performance lower than non-TL direct model)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JHmZb8srCioy",
        "outputId": "ed71509a-fe0f-4a8d-8e81-bde1f8507a7b"
      },
      "source": [
        "valC = get_valC()\n",
        "testC = get_testC()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "phase 2 labelled load time = 0.3309566259384155 min\n",
            "There are 16568 trials with 2 electrodes and 3000 time samples\n",
            "(16568, 2, 3000) (16568,)\n",
            "[7809 1639 4704  689  145 1582]\n",
            "Z-scoring time 7.114246129989624 sec\n",
            "Torch dataset time 0.0020368099212646484 sec = 3.394683202107747e-05 min\n",
            "phase 2 final test set load time = 0.3870764295260111 min\n",
            "There are 25756 trials with 2 electrodes and 3000 time samples\n",
            "Z-scoring time 1.73142409324646 sec\n",
            "Torch dataset time 6.651878356933594e-05 sec = 1.1086463928222657e-06 min\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LkavjcKXCio8"
      },
      "source": [
        "# Channel 0\n",
        "valC0 = Dataset(valC.X[:,0,:].unsqueeze(1),valC.y.long())\n",
        "testC0 = Dataset(testC.X[:,0,:].unsqueeze(1),testC.y.long())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 179
        },
        "id": "adEseQ3UDKIV",
        "outputId": "026eba4f-cf52-44c7-f810-1ac48ada1a3b"
      },
      "source": [
        "# Load the Channel 0, batch_size 1024 model\n",
        "# starting wandb init\n",
        "wandb_run = wandb.init(name = \"AttnSleep_TL_new\", project='cnn-fulltrain', entity='sleep_hacking')\n",
        "\n",
        "# model = clf.module\n",
        "\n",
        "# batch_size\n",
        "lr = 1e-3 #change\n",
        "batch_size = 1024\n",
        "n_epochs = 100\n",
        "\n",
        "channel = 0\n",
        "save_path = \"/content/drive/MyDrive/mne_data/NeuroIPS_Hack_Sleep/AttnSleepTLnew0_checkpoints\"\n",
        "wandb_run.config.update({\"lr\":lr,\"batch_size\":batch_size,\"channel\":channel,\"save_path\":save_path})\n",
        "\n",
        "train_bacc = EpochScoring(\n",
        "    scoring='balanced_accuracy', on_train=True, name='train_bacc',\n",
        "    lower_is_better=False)\n",
        "cp = Checkpoint(monitor = 'train_bacc_best',\n",
        "                f_params = \"params_{last_epoch[epoch]}.pt\", \n",
        "                f_optimizer = \"optimizer_{last_epoch[epoch]}.pt\", \n",
        "                f_criterion = \"criterion_{last_epoch[epoch]}.pt\",\n",
        "                dirname = save_path)\n",
        "train_end_cp = TrainEndCheckpoint(\n",
        "                f_params = \"params_{last_epoch[epoch]}.pt\", \n",
        "                f_optimizer = \"optimizer_{last_epoch[epoch]}.pt\", \n",
        "                f_criterion = \"criterion_{last_epoch[epoch]}.pt\",\n",
        "                dirname=save_path)\n",
        "\n",
        "callbacks = [('train_bal_acc', train_bacc),\n",
        "             (\"checkpoint\",cp),\n",
        "             (\"train_end_cp\",train_end_cp),\n",
        "             (\"wandb\",WandbLogger(wandb_run,save_model=False))\n",
        "            ]\n",
        "\n",
        "clf5 = NNClassifier(\n",
        "  module = AttnSleep,\n",
        "  criterion = weighted_CrossEntropyLoss,\n",
        "  optimizer=torch.optim.Adam,\n",
        "  train_split=None,\n",
        "  iterator_train = DataLoader,\n",
        "  optimizer__lr=lr,\n",
        "  batch_size=batch_size,\n",
        "  callbacks=callbacks,\n",
        "  warm_start = True,\n",
        "  device=device\n",
        ")\n",
        "print(clf5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.12.2<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">AttnSleep_TL_new</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/sleep_hacking/cnn-fulltrain\" target=\"_blank\">https://wandb.ai/sleep_hacking/cnn-fulltrain</a><br/>\n",
              "                Run page: <a href=\"https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/1e0wpixp\" target=\"_blank\">https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/1e0wpixp</a><br/>\n",
              "                Run data is saved locally in <code>/content/wandb/run-20210927_143637-1e0wpixp</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized](\n",
            "  module=<class '__main__.AttnSleep'>,\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ypd4-hXlq0P_"
      },
      "source": [
        "clf5.initialize() # This is important!\n",
        "load_path = \"/content/drive/MyDrive/mne_data/NeuroIPS_Hack_Sleep/AttnSleepTLnew1_checkpoints\"\n",
        "clf5.load_params(\n",
        "    f_params=load_path+'/params_3.pt', f_optimizer=load_path+'/optimizer_3.pt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2MrH83X3DKIY",
        "outputId": "8d8c9b69-ee74-4e6e-a0cf-319a75b91641"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valA0.y\n",
        "y_pred = clf5.predict(valA0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Leaderboard Validation data\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Leaderboard Validation data\n",
            "[[5848   55   23    1    2   81]\n",
            " [  25 1474  119    6    6   42]\n",
            " [  23   43 4657  197   72   43]\n",
            " [   0    0   15  638   50    1]\n",
            " [   0    0    1    1  412    0]\n",
            " [  10   15   27    0    0 1555]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      0.97      0.98      6010\n",
            "           1       0.93      0.88      0.90      1672\n",
            "           2       0.96      0.92      0.94      5035\n",
            "           3       0.76      0.91      0.82       704\n",
            "           4       0.76      1.00      0.86       414\n",
            "           5       0.90      0.97      0.93      1607\n",
            "\n",
            "    accuracy                           0.94     15442\n",
            "   macro avg       0.88      0.94      0.91     15442\n",
            "weighted avg       0.95      0.94      0.95     15442\n",
            "\n",
            "0.923122533570641\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IFW2ufXNDKIY",
        "outputId": "c32ba25d-c1b4-4dca-ac99-a3084b23bc81"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valC0.y\n",
        "y_pred = clf5.predict(valC0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Phase 2 Validation Data before TL training\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Phase 2 Validation Data before TL training\n",
            "[[7235  341   56    9    3  165]\n",
            " [ 319  412  501   67   12  328]\n",
            " [ 117  323 2715  915  353  281]\n",
            " [   5    3   31  198  450    2]\n",
            " [   2    0    0    6  137    0]\n",
            " [ 107  234  400   57   13  771]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.93      0.93      7809\n",
            "           1       0.31      0.25      0.28      1639\n",
            "           2       0.73      0.58      0.65      4704\n",
            "           3       0.16      0.29      0.20       689\n",
            "           4       0.14      0.94      0.25       145\n",
            "           5       0.50      0.49      0.49      1582\n",
            "\n",
            "    accuracy                           0.69     16568\n",
            "   macro avg       0.46      0.58      0.47     16568\n",
            "weighted avg       0.73      0.69      0.71     16568\n",
            "\n",
            "0.5568759416152871\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VCJXeonxDKIZ",
        "outputId": "863dd0b9-d0b7-49ee-cbd8-5ce75b9e9a91"
      },
      "source": [
        "start = time.time()\n",
        "clf5.fit(valC0, y=None, epochs=n_epochs)\n",
        "end = time.time()\n",
        "print(f'time taken : {end-start}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  epoch    train_bacc    train_loss    cp     dur\n",
            "-------  ------------  ------------  ----  ------\n",
            "      1        \u001b[36m0.5004\u001b[0m        \u001b[32m1.2982\u001b[0m     +  2.3097\n",
            "      2        \u001b[36m0.5097\u001b[0m        \u001b[32m0.8621\u001b[0m     +  1.8406\n",
            "      3        \u001b[36m0.5890\u001b[0m        \u001b[32m0.6924\u001b[0m     +  1.9452\n",
            "      4        0.5725        \u001b[32m0.6719\u001b[0m        1.8371\n",
            "      5        \u001b[36m0.6155\u001b[0m        \u001b[32m0.6118\u001b[0m     +  1.8451\n",
            "      6        \u001b[36m0.6579\u001b[0m        \u001b[32m0.5650\u001b[0m     +  1.8641\n",
            "      7        \u001b[36m0.6687\u001b[0m        \u001b[32m0.5269\u001b[0m     +  1.9545\n",
            "      8        \u001b[36m0.6736\u001b[0m        \u001b[32m0.5111\u001b[0m     +  1.8410\n",
            "      9        \u001b[36m0.6981\u001b[0m        \u001b[32m0.4863\u001b[0m     +  1.8510\n",
            "     10        \u001b[36m0.7148\u001b[0m        \u001b[32m0.4744\u001b[0m     +  1.8334\n",
            "     11        0.7032        \u001b[32m0.4693\u001b[0m        1.8448\n",
            "     12        \u001b[36m0.7222\u001b[0m        \u001b[32m0.4613\u001b[0m     +  1.9981\n",
            "     13        \u001b[36m0.7257\u001b[0m        \u001b[32m0.4486\u001b[0m     +  1.8425\n",
            "     14        \u001b[36m0.7500\u001b[0m        \u001b[32m0.4334\u001b[0m     +  1.8387\n",
            "     15        0.7425        \u001b[32m0.4306\u001b[0m        1.8436\n",
            "     16        \u001b[36m0.7503\u001b[0m        \u001b[32m0.4230\u001b[0m     +  1.9559\n",
            "     17        0.7484        \u001b[32m0.4131\u001b[0m        1.8315\n",
            "     18        \u001b[36m0.7663\u001b[0m        \u001b[32m0.4023\u001b[0m     +  1.8753\n",
            "     19        \u001b[36m0.7804\u001b[0m        \u001b[32m0.3895\u001b[0m     +  1.8382\n",
            "     20        0.7762        \u001b[32m0.3866\u001b[0m        1.8328\n",
            "     21        \u001b[36m0.7866\u001b[0m        \u001b[32m0.3787\u001b[0m     +  1.9450\n",
            "     22        0.7781        \u001b[32m0.3723\u001b[0m        1.8381\n",
            "     23        \u001b[36m0.7910\u001b[0m        \u001b[32m0.3697\u001b[0m     +  1.8325\n",
            "     24        \u001b[36m0.7976\u001b[0m        \u001b[32m0.3585\u001b[0m     +  1.8711\n",
            "     25        \u001b[36m0.8049\u001b[0m        \u001b[32m0.3475\u001b[0m     +  1.8395\n",
            "     26        0.8006        \u001b[32m0.3410\u001b[0m        1.9607\n",
            "     27        0.8003        0.3426        1.8390\n",
            "     28        \u001b[36m0.8256\u001b[0m        \u001b[32m0.3334\u001b[0m     +  1.8370\n",
            "     29        0.7888        0.3506        1.8417\n",
            "     30        0.7786        0.3438        1.9780\n",
            "     31        0.8084        \u001b[32m0.3246\u001b[0m        1.8334\n",
            "     32        0.8101        0.3256        1.8368\n",
            "     33        \u001b[36m0.8266\u001b[0m        \u001b[32m0.3046\u001b[0m     +  1.8468\n",
            "     34        \u001b[36m0.8465\u001b[0m        \u001b[32m0.2881\u001b[0m     +  1.9839\n",
            "     35        0.8412        \u001b[32m0.2838\u001b[0m        1.8427\n",
            "     36        \u001b[36m0.8476\u001b[0m        \u001b[32m0.2770\u001b[0m     +  1.8710\n",
            "     37        \u001b[36m0.8588\u001b[0m        \u001b[32m0.2562\u001b[0m     +  1.8433\n",
            "     38        0.8570        0.2565        1.8372\n",
            "     39        \u001b[36m0.8714\u001b[0m        \u001b[32m0.2467\u001b[0m     +  1.9590\n",
            "     40        0.8681        \u001b[32m0.2386\u001b[0m        1.8403\n",
            "     41        \u001b[36m0.8738\u001b[0m        \u001b[32m0.2345\u001b[0m     +  1.8485\n",
            "     42        \u001b[36m0.8762\u001b[0m        0.2397     +  1.8645\n",
            "     43        0.8678        0.2402        1.8497\n",
            "     44        0.8501        0.2490        1.8452\n",
            "     45        0.8410        0.2547        1.9471\n",
            "     46        0.8646        \u001b[32m0.2291\u001b[0m        1.8401\n",
            "     47        0.8686        0.2302        1.8449\n",
            "     48        \u001b[36m0.8891\u001b[0m        \u001b[32m0.2123\u001b[0m     +  1.8672\n",
            "     49        \u001b[36m0.8980\u001b[0m        \u001b[32m0.1937\u001b[0m     +  1.8372\n",
            "     50        0.8905        0.2022        1.8373\n",
            "     51        \u001b[36m0.8982\u001b[0m        0.1954     +  1.8353\n",
            "     52        0.8932        0.1995        1.9573\n",
            "     53        0.8960        \u001b[32m0.1900\u001b[0m        1.8615\n",
            "     54        \u001b[36m0.9019\u001b[0m        \u001b[32m0.1827\u001b[0m     +  1.8342\n",
            "     55        0.9002        \u001b[32m0.1767\u001b[0m        1.8353\n",
            "     56        \u001b[36m0.9124\u001b[0m        \u001b[32m0.1644\u001b[0m     +  1.9614\n",
            "     57        0.9015        \u001b[32m0.1632\u001b[0m        1.8372\n",
            "     58        0.9026        0.1749        1.8392\n",
            "     59        0.9118        \u001b[32m0.1594\u001b[0m        1.8654\n",
            "     60        \u001b[36m0.9194\u001b[0m        \u001b[32m0.1516\u001b[0m     +  1.8433\n",
            "     61        0.9133        0.1551        1.8475\n",
            "     62        \u001b[36m0.9197\u001b[0m        \u001b[32m0.1433\u001b[0m     +  1.8352\n",
            "     63        \u001b[36m0.9246\u001b[0m        \u001b[32m0.1358\u001b[0m     +  1.9538\n",
            "     64        \u001b[36m0.9350\u001b[0m        \u001b[32m0.1265\u001b[0m     +  1.8436\n",
            "     65        \u001b[36m0.9354\u001b[0m        0.1276     +  1.8628\n",
            "     66        \u001b[36m0.9380\u001b[0m        \u001b[32m0.1204\u001b[0m     +  1.8464\n",
            "     67        \u001b[36m0.9380\u001b[0m        0.1242     +  1.8453\n",
            "     68        \u001b[36m0.9449\u001b[0m        \u001b[32m0.1045\u001b[0m     +  1.8429\n",
            "     69        0.9447        0.1090        1.9483\n",
            "     70        0.9406        0.1104        1.8267\n",
            "     71        \u001b[36m0.9510\u001b[0m        \u001b[32m0.1026\u001b[0m     +  1.8604\n",
            "     72        0.9459        0.1085        1.8361\n",
            "     73        0.9455        0.1058        1.8371\n",
            "     74        0.9451        0.1033        1.8381\n",
            "     75        0.9422        \u001b[32m0.1012\u001b[0m        1.9458\n",
            "     76        0.9495        0.1043        1.8317\n",
            "     77        0.9435        0.1076        1.8577\n",
            "     78        0.9422        0.1129        1.8404\n",
            "     79        0.9408        0.1103        1.8298\n",
            "     80        \u001b[36m0.9517\u001b[0m        \u001b[32m0.0999\u001b[0m     +  1.8539\n",
            "     81        \u001b[36m0.9528\u001b[0m        \u001b[32m0.0976\u001b[0m     +  1.8620\n",
            "     82        0.9523        \u001b[32m0.0923\u001b[0m        1.9492\n",
            "     83        0.9435        \u001b[32m0.0915\u001b[0m        1.8655\n",
            "     84        \u001b[36m0.9534\u001b[0m        0.0917     +  1.8362\n",
            "     85        \u001b[36m0.9542\u001b[0m        \u001b[32m0.0884\u001b[0m     +  1.8357\n",
            "     86        0.9499        0.0947        1.8417\n",
            "     87        0.9525        0.0896        1.8403\n",
            "     88        \u001b[36m0.9554\u001b[0m        \u001b[32m0.0848\u001b[0m     +  1.9605\n",
            "     89        0.9466        0.0889        1.8730\n",
            "     90        0.9533        0.0894        1.8448\n",
            "     91        0.9514        0.0959        1.8387\n",
            "     92        0.9456        0.0905        1.8365\n",
            "     93        0.9544        0.0901        1.8387\n",
            "     94        0.9501        0.0943        1.8365\n",
            "     95        \u001b[36m0.9631\u001b[0m        0.0883     +  1.9931\n",
            "     96        0.9572        \u001b[32m0.0829\u001b[0m        1.8389\n",
            "     97        0.9572        \u001b[32m0.0800\u001b[0m        1.8419\n",
            "     98        0.9589        \u001b[32m0.0775\u001b[0m        1.8429\n",
            "     99        0.9609        \u001b[32m0.0746\u001b[0m        1.8369\n",
            "    100        \u001b[36m0.9649\u001b[0m        \u001b[32m0.0715\u001b[0m     +  1.8634\n",
            "time taken : 191.9069094657898\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zNOSUEiVDKIa",
        "outputId": "587553e9-c2dd-4253-95be-f1ef0ccbc916"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valC0.y\n",
        "y_pred = clf5.predict(valC0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Phase 2 Validation data after TL training on Phase 2 val\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Phase 2 Validation data after TL training on Phase 2 val\n",
            "[[7790    7    3    1    1    7]\n",
            " [  39 1476   42    0    0   82]\n",
            " [   9   20 4415   21    1  238]\n",
            " [   0    0   42  647    0    0]\n",
            " [   0    0    0    1  144    0]\n",
            " [   9    0    5    0    0 1568]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      1.00      1.00      7809\n",
            "           1       0.98      0.90      0.94      1639\n",
            "           2       0.98      0.94      0.96      4704\n",
            "           3       0.97      0.94      0.95       689\n",
            "           4       0.99      0.99      0.99       145\n",
            "           5       0.83      0.99      0.90      1582\n",
            "\n",
            "    accuracy                           0.97     16568\n",
            "   macro avg       0.96      0.96      0.96     16568\n",
            "weighted avg       0.97      0.97      0.97     16568\n",
            "\n",
            "0.9529875599569695\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DwzxSTQ3DKIb",
        "outputId": "6d948815-492f-4b56-be6c-6261010e08dc"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = valA0.y\n",
        "y_pred = clf5.predict(valA0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on Leaderboard Validation data after TL training on Phase 2 val\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on Leaderboard Validation data after TL training on Phase 2 val\n",
            "[[5451  201   97   10    2  249]\n",
            " [ 249  469  453    0    2  499]\n",
            " [ 103  253 4124   20    2  533]\n",
            " [   1    2  612   83    2    4]\n",
            " [   0    0  236  140   38    0]\n",
            " [ 140   65  113    0    2 1287]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.91      0.91      6010\n",
            "           1       0.47      0.28      0.35      1672\n",
            "           2       0.73      0.82      0.77      5035\n",
            "           3       0.33      0.12      0.17       704\n",
            "           4       0.79      0.09      0.16       414\n",
            "           5       0.50      0.80      0.62      1607\n",
            "\n",
            "    accuracy                           0.74     15442\n",
            "   macro avg       0.62      0.50      0.50     15442\n",
            "weighted avg       0.74      0.74      0.72     15442\n",
            "\n",
            "0.6340654649550557\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cM32NwnuDKIc",
        "outputId": "653c139f-6988-4923-810b-e8bbb124e5d0"
      },
      "source": [
        "#checking if loaded properly\n",
        "y_true = trainA0.y\n",
        "y_pred = clf5.predict(trainA0.X)\n",
        "\n",
        "print(\"From Last Checkpointed model - Prediction on original Group 1 training data after TL training on Phase 2 Val data\")\n",
        "print(confusion_matrix(y_true.cpu(), y_pred))\n",
        "print(classification_report(y_true.cpu(), y_pred))\n",
        "cohen_kappa = cohen_kappa_score(y_true.cpu(), y_pred)\n",
        "print(cohen_kappa)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From Last Checkpointed model - Prediction on original Group 1 training data after TL training on Phase 2 Val data\n",
            "[[21626   662   436    26    12  1281]\n",
            " [ 1440   991  2215     7     0  3288]\n",
            " [  850  1001 30391   255     7  3479]\n",
            " [   26    16  4133   974    81    17]\n",
            " [    0     1  1372  1318   365     1]\n",
            " [ 1244   440  1988    30     2 10570]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.86      0.90      0.88     24043\n",
            "           1       0.32      0.12      0.18      7941\n",
            "           2       0.75      0.84      0.79     35983\n",
            "           3       0.37      0.19      0.25      5247\n",
            "           4       0.78      0.12      0.21      3057\n",
            "           5       0.57      0.74      0.64     14274\n",
            "\n",
            "    accuracy                           0.72     90545\n",
            "   macro avg       0.61      0.49      0.49     90545\n",
            "weighted avg       0.69      0.72      0.69     90545\n",
            "\n",
            "0.6018683529800133\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eSfOFw0gDKId"
      },
      "source": [
        "Good Example of Catastrophic Forgetting, but group staying at around 60% is interesting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Ha0Q2SqDKId",
        "outputId": "016e7513-3e3c-4589-c586-23f717cf516e"
      },
      "source": [
        "predict_leaderboard_unlabelled(clf=clf5,save_fname=\"AttnSleep_TL_final1\", x_test_data = testC.X[:,channel,:].unsqueeze(1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([25756, 1, 3000])\n",
            "Checking if all classes have been predicted\n",
            "[0 1 2 3 4 5]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I8m8Wd4EDKIe"
      },
      "source": [
        "This model got 63.8% on the Sleep Leaderboard. Without the TL it was at 60%.. Not such a great improvement.. Seems like Overfitting on the 5 subject data wasn't enough to generalize to all the leaderboard data.."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443,
          "referenced_widgets": [
            "bcc1bf44d476410fb619ffdca6fc31ec",
            "1c2e25dd1fe543eb8b5b0f05ca83f627",
            "80ed1c51d00142e1a7bd2276b8f1228b",
            "ee7d3c67db304143966b2a7a4f4a258c",
            "5fe67647e3d94150a771abdbbedb855c",
            "d3ca02fb63c0483db6fd0f9bd8166b32",
            "ed99830578fc456d88207a3d1e4fc531",
            "4b80d35aba90474592c5e389fc78c471"
          ]
        },
        "id": "SXqYAS77DKIe",
        "outputId": "818164fc-5501-4481-dcfc-b5e86e2d520c"
      },
      "source": [
        "wandb.finish()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<br/>Waiting for W&B process to finish, PID 4460<br/>Program ended successfully."
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bcc1bf44d476410fb619ffdca6fc31ec",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "VBox(children=(Label(value=' 0.00MB of 0.00MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Find user logs for this run at: <code>/content/wandb/run-20210927_143637-1e0wpixp/logs/debug.log</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Find internal logs for this run at: <code>/content/wandb/run-20210927_143637-1e0wpixp/logs/debug-internal.log</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<h3>Run summary:</h3><br/><style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    </style><table class=\"wandb\">\n",
              "<tr><td>dur</td><td>1.86337</td></tr><tr><td>train_bacc</td><td>0.96495</td></tr><tr><td>train_loss</td><td>0.07148</td></tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<h3>Run history:</h3><br/><style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    </style><table class=\"wandb\">\n",
              "<tr><td>dur</td><td>█▃▂▁▁▁▃▂▃▁▃▁▁▃▂▃▁▁▁▁▁▁▃▂▁▁▁▃▁▁▁▁▃▁▁▂▁▁▁▁</td></tr><tr><td>train_bacc</td><td>▁▂▃▄▄▄▅▅▅▅▆▆▆▆▆▇▇▆▆▇▇▇▇▇▇███████████████</td></tr><tr><td>train_loss</td><td>█▅▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr></table><br/>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                    <br/>Synced <strong style=\"color:#cdcd00\">AttnSleep_TL_new</strong>: <a href=\"https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/1e0wpixp\" target=\"_blank\">https://wandb.ai/sleep_hacking/cnn-fulltrain/runs/1e0wpixp</a><br/>\n",
              "                "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "91AoT14vWFJ9"
      },
      "source": [
        "I only got around 46.77% and 45.38% when model with TL on val1 then val2 and model with only TL on val2 respectively on the final test phase data - These were overfitted models which was caused this. Lesser around the model after the 3rd epoch would've been better."
      ]
    }
  ]
}