3. CIFAR100_ResNet_Scratch_DataAugmentation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Content:\n",
    "- Test a different network architecuture, ResNet\n",
    "- DataAugmentation: RandomHorizontalFlip, RandomCrop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### code to build the network ist from:\n",
    "Source: https://www.kaggle.com/code/toygarr/resnet-implementation-for-image-classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision.datasets import CIFAR100\n",
    "from torchvision.utils import make_grid\n",
    "from torch.utils.data.dataloader import DataLoader\n",
    "from torch.utils.data import random_split, ConcatDataset\n",
    "import torchvision.transforms as tt\n",
    "from torchvision import transforms\n",
    "import pickle\n",
    "from torch.optim import lr_scheduler\n",
    "from torchsummary import summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training images train data set:  40000\n",
      "Number of validation images in train dataset:  10000\n",
      "Number of test images test data set:  10000\n"
     ]
    }
   ],
   "source": [
    "train_transform = tt.Compose([\n",
    "    tt.RandomHorizontalFlip(),\n",
    "    tt.RandomCrop(32, padding=4, padding_mode=\"reflect\"),\n",
    "    tt.ToTensor(),\n",
    "    tt.Normalize((0.5074,0.4867,0.4411),(0.2011,0.1987,0.2025))\n",
    "])\n",
    "\n",
    "test_transform = tt.Compose([\n",
    "    tt.ToTensor(),\n",
    "    tt.Normalize((0.5074,0.4867,0.4411),(0.2011,0.1987,0.2025))\n",
    "])\n",
    "\n",
    "train_dataset = CIFAR100(root=\"./\", train=True, download=False, transform=train_transform)\n",
    "test_dataset = CIFAR100(root=\"./\", train=False, download=False, transform=test_transform)\n",
    "\n",
    "# Aufteilen des Trainingsdatensatzes in Trainings- und Validierungsdatensatz\n",
    "val_size = int(0.2 * len(train_dataset))  \n",
    "train_size = len(train_dataset) - val_size\n",
    "train_dataset, val_dataset = random_split(train_dataset, [train_size, val_size])\n",
    "\n",
    "\n",
    "batch_size = 64\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "valid_loader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "print('Number of training images train data set: ', len(train_dataset))\n",
    "print('Number of validation images in train dataset: ', len(val_dataset))\n",
    "print('Number of test images test data set: ', len(test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['apple', 'aquarium_fish', 'baby', 'bear', 'beaver', 'bed', 'bee', 'beetle', 'bicycle', 'bottle', 'bowl', 'boy', 'bridge', 'bus', 'butterfly', 'camel', 'can', 'castle', 'caterpillar', 'cattle', 'chair', 'chimpanzee', 'clock', 'cloud', 'cockroach', 'couch', 'crab', 'crocodile', 'cup', 'dinosaur', 'dolphin', 'elephant', 'flatfish', 'forest', 'fox', 'girl', 'hamster', 'house', 'kangaroo', 'keyboard', 'lamp', 'lawn_mower', 'leopard', 'lion', 'lizard', 'lobster', 'man', 'maple_tree', 'motorcycle', 'mountain', 'mouse', 'mushroom', 'oak_tree', 'orange', 'orchid', 'otter', 'palm_tree', 'pear', 'pickup_truck', 'pine_tree', 'plain', 'plate', 'poppy', 'porcupine', 'possum', 'rabbit', 'raccoon', 'ray', 'road', 'rocket', 'rose', 'sea', 'seal', 'shark', 'shrew', 'skunk', 'skyscraper', 'snail', 'snake', 'spider', 'squirrel', 'streetcar', 'sunflower', 'sweet_pepper', 'table', 'tank', 'telephone', 'television', 'tiger', 'tractor', 'train', 'trout', 'tulip', 'turtle', 'wardrobe', 'whale', 'willow_tree', 'wolf', 'woman', 'worm']\n"
     ]
    }
   ],
   "source": [
    "# Load the label names from metadata file\n",
    "with open('cifar-100-python/meta', 'rb') as f:\n",
    "    meta_dict = pickle.load(f, encoding='bytes')\n",
    "fine_label_names = [t.decode('utf8') for t in meta_dict[b'fine_label_names']]\n",
    "print(fine_label_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Model\n",
    "\n",
    "#### architecture source: https://www.kaggle.com/code/toygarr/resnet-implementation-for-image-classification\n",
    "\n",
    "![image.png](https://open-instruction.com/loading/2021/05/q.png)\n",
    "\n",
    "#### Shortcut connections source: https://arxiv.org/pdf/1512.03385.pdf\n",
    "#### Shortcut connections source: https://towardsdatascience.com/building-a-residual-network-with-pytorch-df2f6937053b\n",
    "#### Shortcut connections source https://towardsdatascience.com/residual-networks-resnets-cb474c7c834a#:~:text=The%20identity%20block%20is%20the,is%20the%20%E2%80%9Cmain%20path%E2%80%9D.\n",
    "\n",
    "![image.png](https://miro.medium.com/v2/resize:fit:4800/format:webp/1*dhQQdqZ_XciBou1yAPL8ow.jpeg)\n",
    "\n",
    "- The skip-connections help to address the Vanishing Gradient problem. They also make it easy for a ResNet block to learn an identity function.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BaseModel(nn.Module):\n",
    "    def training_step(self,batch):\n",
    "        images, labels = batch\n",
    "        out = self(images)\n",
    "        loss = F.cross_entropy(out,labels)\n",
    "        return loss\n",
    "    \n",
    "    def validation_step(self,batch):\n",
    "        images, labels = batch\n",
    "        out = self(images)\n",
    "        loss = F.cross_entropy(out,labels)\n",
    "        acc = accuracy(out,labels)\n",
    "        return {\"val_loss\":loss.detach(),\"val_acc\":acc}\n",
    "    \n",
    "    def validation_epoch_end(self,outputs):\n",
    "        batch_losses = [loss[\"val_loss\"] for loss in outputs]\n",
    "        loss = torch.stack(batch_losses).mean()\n",
    "        batch_accuracy = [accuracy[\"val_acc\"] for accuracy in outputs]\n",
    "        acc = torch.stack(batch_accuracy).mean()\n",
    "        return {\"val_loss\":loss.item(),\"val_acc\":acc.item()}\n",
    "    \n",
    "    def epoch_end(self, epoch, result):\n",
    "        print(\"Epoch [{}], last_lr: {:.5f}, train_loss: {:.4f}, val_loss: {:.4f}, val_acc: {:.4f}\".format(\n",
    "            epoch, result['lrs'][-1], result['train_loss'], result['val_loss'], result['val_acc']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_shortcut(in_channel, out_channel, stride):\n",
    "    layers = [nn.Conv2d(in_channel, out_channel, kernel_size=(1,1), stride=(stride, stride)),\n",
    "             nn.BatchNorm2d(out_channel)]\n",
    "    return nn.Sequential(*layers)\n",
    "\n",
    "def block(in_channel, out_channel, k_size,stride, conv=False):\n",
    "    layers = None\n",
    "    \n",
    "    first_layers = [nn.Conv2d(in_channel,out_channel[0], kernel_size=(1,1),stride=(1,1)),\n",
    "                    nn.BatchNorm2d(out_channel[0]),\n",
    "                    nn.ReLU(inplace=True)]\n",
    "    if conv:\n",
    "        first_layers[0].stride=(stride,stride)\n",
    "    \n",
    "    second_layers = [nn.Conv2d(out_channel[0], out_channel[1], kernel_size=(k_size, k_size), stride=(1,1), padding=1),\n",
    "                    nn.BatchNorm2d(out_channel[1])]\n",
    "\n",
    "    layers = first_layers + second_layers\n",
    "    \n",
    "    return nn.Sequential(*layers)\n",
    "    \n",
    "\n",
    "class ResNet(BaseModel):\n",
    "    \n",
    "    def __init__(self, in_channels, num_classes):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.stg1 = nn.Sequential(\n",
    "                                   nn.Conv2d(in_channels=in_channels, out_channels=64, kernel_size=(3),\n",
    "                                             stride=(1), padding=1),\n",
    "                                   nn.BatchNorm2d(64),\n",
    "                                   nn.ReLU(inplace=True),\n",
    "                                   nn.MaxPool2d(kernel_size=3, stride=2))\n",
    "        \n",
    "        ##stage 2\n",
    "        self.convShortcut2 = conv_shortcut(64,256,1)        \n",
    "        self.conv2 = block(64,[64,256],3,1,conv=True)\n",
    "        self.ident2 = block(256,[64,256],3,1)\n",
    "\n",
    "        \n",
    "        ##stage 3\n",
    "        self.convShortcut3 = conv_shortcut(256,512,2)       \n",
    "        self.conv3 = block(256,[128,512],3,2,conv=True)\n",
    "        self.ident3 = block(512,[128,512],3,2)\n",
    "\n",
    "        \n",
    "        ##stage 4\n",
    "        self.convShortcut4 = conv_shortcut(512,1024,2)     \n",
    "        self.conv4 = block(512,[256,1024],3,2,conv=True)\n",
    "        self.ident4 = block(1024,[256,1024],3,2)\n",
    "        \n",
    "        \n",
    "        ##Classify\n",
    "        self.classifier = nn.Sequential(\n",
    "                                       nn.AvgPool2d(kernel_size=(4)),\n",
    "                                       nn.Flatten(),\n",
    "                                       nn.Linear(1024, num_classes))\n",
    "        \n",
    "    def forward(self,inputs):\n",
    "        out = self.stg1(inputs)\n",
    "        \n",
    "        #stage 2\n",
    "        out = F.relu(self.conv2(out) + self.convShortcut2(out))\n",
    "        out = F.relu(self.ident2(out) + out)\n",
    "        out = F.relu(self.ident2(out) + out)\n",
    "        out = F.relu(self.ident2(out) + out)\n",
    "        \n",
    "        #stage3\n",
    "        out = F.relu(self.conv3(out) + (self.convShortcut3(out)))\n",
    "        out = F.relu(self.ident3(out) + out)\n",
    "        out = F.relu(self.ident3(out) + out)\n",
    "        out = F.relu(self.ident3(out) + out)\n",
    "        out = F.relu(self.ident3(out) + out)\n",
    "        \n",
    "        #stage4             \n",
    "        out = F.relu(self.conv4(out) + (self.convShortcut4(out)))\n",
    "        out = F.relu(self.ident4(out) + out)\n",
    "        out = F.relu(self.ident4(out) + out)\n",
    "        out = F.relu(self.ident4(out) + out)\n",
    "        out = F.relu(self.ident4(out) + out)\n",
    "        out = F.relu(self.ident4(out) + out)\n",
    "        out = F.relu(self.ident4(out) + out)\n",
    "        \n",
    "        #Classify\n",
    "        out = self.classifier(out)#100x1024\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Device configuration\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ResNet(3,100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Take a look at the architecture and the number of parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 64, 32, 32]           1,792\n",
      "       BatchNorm2d-2           [-1, 64, 32, 32]             128\n",
      "              ReLU-3           [-1, 64, 32, 32]               0\n",
      "         MaxPool2d-4           [-1, 64, 15, 15]               0\n",
      "            Conv2d-5           [-1, 64, 15, 15]           4,160\n",
      "       BatchNorm2d-6           [-1, 64, 15, 15]             128\n",
      "              ReLU-7           [-1, 64, 15, 15]               0\n",
      "            Conv2d-8          [-1, 256, 15, 15]         147,712\n",
      "       BatchNorm2d-9          [-1, 256, 15, 15]             512\n",
      "           Conv2d-10          [-1, 256, 15, 15]          16,640\n",
      "      BatchNorm2d-11          [-1, 256, 15, 15]             512\n",
      "           Conv2d-12           [-1, 64, 15, 15]          16,448\n",
      "      BatchNorm2d-13           [-1, 64, 15, 15]             128\n",
      "             ReLU-14           [-1, 64, 15, 15]               0\n",
      "           Conv2d-15          [-1, 256, 15, 15]         147,712\n",
      "      BatchNorm2d-16          [-1, 256, 15, 15]             512\n",
      "           Conv2d-17           [-1, 64, 15, 15]          16,448\n",
      "      BatchNorm2d-18           [-1, 64, 15, 15]             128\n",
      "             ReLU-19           [-1, 64, 15, 15]               0\n",
      "           Conv2d-20          [-1, 256, 15, 15]         147,712\n",
      "      BatchNorm2d-21          [-1, 256, 15, 15]             512\n",
      "           Conv2d-22           [-1, 64, 15, 15]          16,448\n",
      "      BatchNorm2d-23           [-1, 64, 15, 15]             128\n",
      "             ReLU-24           [-1, 64, 15, 15]               0\n",
      "           Conv2d-25          [-1, 256, 15, 15]         147,712\n",
      "      BatchNorm2d-26          [-1, 256, 15, 15]             512\n",
      "           Conv2d-27            [-1, 128, 8, 8]          32,896\n",
      "      BatchNorm2d-28            [-1, 128, 8, 8]             256\n",
      "             ReLU-29            [-1, 128, 8, 8]               0\n",
      "           Conv2d-30            [-1, 512, 8, 8]         590,336\n",
      "      BatchNorm2d-31            [-1, 512, 8, 8]           1,024\n",
      "           Conv2d-32            [-1, 512, 8, 8]         131,584\n",
      "      BatchNorm2d-33            [-1, 512, 8, 8]           1,024\n",
      "           Conv2d-34            [-1, 128, 8, 8]          65,664\n",
      "      BatchNorm2d-35            [-1, 128, 8, 8]             256\n",
      "             ReLU-36            [-1, 128, 8, 8]               0\n",
      "           Conv2d-37            [-1, 512, 8, 8]         590,336\n",
      "      BatchNorm2d-38            [-1, 512, 8, 8]           1,024\n",
      "           Conv2d-39            [-1, 128, 8, 8]          65,664\n",
      "      BatchNorm2d-40            [-1, 128, 8, 8]             256\n",
      "             ReLU-41            [-1, 128, 8, 8]               0\n",
      "           Conv2d-42            [-1, 512, 8, 8]         590,336\n",
      "      BatchNorm2d-43            [-1, 512, 8, 8]           1,024\n",
      "           Conv2d-44            [-1, 128, 8, 8]          65,664\n",
      "      BatchNorm2d-45            [-1, 128, 8, 8]             256\n",
      "             ReLU-46            [-1, 128, 8, 8]               0\n",
      "           Conv2d-47            [-1, 512, 8, 8]         590,336\n",
      "      BatchNorm2d-48            [-1, 512, 8, 8]           1,024\n",
      "           Conv2d-49            [-1, 128, 8, 8]          65,664\n",
      "      BatchNorm2d-50            [-1, 128, 8, 8]             256\n",
      "             ReLU-51            [-1, 128, 8, 8]               0\n",
      "           Conv2d-52            [-1, 512, 8, 8]         590,336\n",
      "      BatchNorm2d-53            [-1, 512, 8, 8]           1,024\n",
      "           Conv2d-54            [-1, 256, 4, 4]         131,328\n",
      "      BatchNorm2d-55            [-1, 256, 4, 4]             512\n",
      "             ReLU-56            [-1, 256, 4, 4]               0\n",
      "           Conv2d-57           [-1, 1024, 4, 4]       2,360,320\n",
      "      BatchNorm2d-58           [-1, 1024, 4, 4]           2,048\n",
      "           Conv2d-59           [-1, 1024, 4, 4]         525,312\n",
      "      BatchNorm2d-60           [-1, 1024, 4, 4]           2,048\n",
      "           Conv2d-61            [-1, 256, 4, 4]         262,400\n",
      "      BatchNorm2d-62            [-1, 256, 4, 4]             512\n",
      "             ReLU-63            [-1, 256, 4, 4]               0\n",
      "           Conv2d-64           [-1, 1024, 4, 4]       2,360,320\n",
      "      BatchNorm2d-65           [-1, 1024, 4, 4]           2,048\n",
      "           Conv2d-66            [-1, 256, 4, 4]         262,400\n",
      "      BatchNorm2d-67            [-1, 256, 4, 4]             512\n",
      "             ReLU-68            [-1, 256, 4, 4]               0\n",
      "           Conv2d-69           [-1, 1024, 4, 4]       2,360,320\n",
      "      BatchNorm2d-70           [-1, 1024, 4, 4]           2,048\n",
      "           Conv2d-71            [-1, 256, 4, 4]         262,400\n",
      "      BatchNorm2d-72            [-1, 256, 4, 4]             512\n",
      "             ReLU-73            [-1, 256, 4, 4]               0\n",
      "           Conv2d-74           [-1, 1024, 4, 4]       2,360,320\n",
      "      BatchNorm2d-75           [-1, 1024, 4, 4]           2,048\n",
      "           Conv2d-76            [-1, 256, 4, 4]         262,400\n",
      "      BatchNorm2d-77            [-1, 256, 4, 4]             512\n",
      "             ReLU-78            [-1, 256, 4, 4]               0\n",
      "           Conv2d-79           [-1, 1024, 4, 4]       2,360,320\n",
      "      BatchNorm2d-80           [-1, 1024, 4, 4]           2,048\n",
      "           Conv2d-81            [-1, 256, 4, 4]         262,400\n",
      "      BatchNorm2d-82            [-1, 256, 4, 4]             512\n",
      "             ReLU-83            [-1, 256, 4, 4]               0\n",
      "           Conv2d-84           [-1, 1024, 4, 4]       2,360,320\n",
      "      BatchNorm2d-85           [-1, 1024, 4, 4]           2,048\n",
      "           Conv2d-86            [-1, 256, 4, 4]         262,400\n",
      "      BatchNorm2d-87            [-1, 256, 4, 4]             512\n",
      "             ReLU-88            [-1, 256, 4, 4]               0\n",
      "           Conv2d-89           [-1, 1024, 4, 4]       2,360,320\n",
      "      BatchNorm2d-90           [-1, 1024, 4, 4]           2,048\n",
      "        AvgPool2d-91           [-1, 1024, 1, 1]               0\n",
      "          Flatten-92                 [-1, 1024]               0\n",
      "           Linear-93                  [-1, 100]         102,500\n",
      "================================================================\n",
      "Total params: 22,927,972\n",
      "Trainable params: 22,927,972\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 13.93\n",
      "Params size (MB): 87.46\n",
      "Estimated Total Size (MB): 101.41\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summary(model, (3, 32, 32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Same training function like always:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, criterion, optimizer, scheduler, num_epochs):\n",
    "    train_loss_list = []\n",
    "    val_loss_list = []\n",
    "    epoch_list = []\n",
    "\n",
    "    total_step = len(train_loader)\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        epoch_train_loss = 0.0\n",
    "        \n",
    "        model.train()             # activate dropout\n",
    "        for i, (images, labels) in enumerate(train_loader):\n",
    "            # Move tensors to the configured device\n",
    "            images = images.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            # Forward pass\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "\n",
    "            # Backward and optimize\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            # Add up the training loss for this batch\n",
    "            epoch_train_loss += loss.item()\n",
    "\n",
    "        # Compute the average training loss for this epoch\n",
    "        epoch_train_loss /= len(train_loader)\n",
    "\n",
    "        # Append the loss to the train_loss_list\n",
    "        train_loss_list.append(epoch_train_loss)\n",
    "        epoch_list.append(epoch+1)\n",
    "        print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, i+1, total_step, epoch_train_loss))\n",
    "\n",
    "        # Validation\n",
    "        #model.eval()           # deactivate dropout\n",
    "        with torch.no_grad():\n",
    "            correct = 0\n",
    "            total = 0\n",
    "            val_loss = 0\n",
    "            for images, labels in valid_loader:\n",
    "                images = images.to(device)\n",
    "                labels = labels.to(device)\n",
    "                outputs = model(images)\n",
    "                loss = criterion(outputs, labels)\n",
    "                val_loss += loss.item() * images.size(0)\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                total += labels.size(0)\n",
    "                correct += (predicted == labels).sum().item()\n",
    "                del images, labels, outputs\n",
    "\n",
    "            # Append the average validation loss to the val_loss_list\n",
    "            val_loss /= len(valid_loader.dataset)\n",
    "            val_loss_list.append(val_loss)\n",
    "            print('Accuracy of the network on the {} validation images: {} %'.format(5000, 100 * correct / total))\n",
    "         \n",
    "        # in each epoch call scheduler \n",
    "        scheduler.step()\n",
    "        current_lr = optimizer.param_groups[0]['lr']\n",
    "        print(\"Current learning rate: {}\".format(current_lr))\n",
    "\n",
    "    # Plot the train and validation loss\n",
    "    import matplotlib.pyplot as plt\n",
    "    plt.plot(epoch_list, train_loss_list, label='Train Loss')\n",
    "    plt.plot(epoch_list, val_loss_list, label='Validation Loss')\n",
    "    plt.legend()\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 100\n",
    "learning_rate = 0.05\n",
    "\n",
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay = 0.005, momentum = 0.9)  \n",
    "\n",
    "# Train the model\n",
    "total_step = len(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "step_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=35, gamma = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/100], Step [625/625], Loss: 5.1809\n",
      "Accuracy of the network on the 5000 validation images: 2.52 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [2/100], Step [625/625], Loss: 4.2243\n",
      "Accuracy of the network on the 5000 validation images: 5.79 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [3/100], Step [625/625], Loss: 4.0003\n",
      "Accuracy of the network on the 5000 validation images: 7.83 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [4/100], Step [625/625], Loss: 3.8932\n",
      "Accuracy of the network on the 5000 validation images: 10.57 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [5/100], Step [625/625], Loss: 3.7832\n",
      "Accuracy of the network on the 5000 validation images: 11.42 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [6/100], Step [625/625], Loss: 3.7179\n",
      "Accuracy of the network on the 5000 validation images: 12.87 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [7/100], Step [625/625], Loss: 3.6630\n",
      "Accuracy of the network on the 5000 validation images: 12.58 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [8/100], Step [625/625], Loss: 3.6187\n",
      "Accuracy of the network on the 5000 validation images: 14.47 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [9/100], Step [625/625], Loss: 3.5801\n",
      "Accuracy of the network on the 5000 validation images: 14.76 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [10/100], Step [625/625], Loss: 3.5193\n",
      "Accuracy of the network on the 5000 validation images: 16.06 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [11/100], Step [625/625], Loss: 3.4691\n",
      "Accuracy of the network on the 5000 validation images: 17.38 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [12/100], Step [625/625], Loss: 3.4173\n",
      "Accuracy of the network on the 5000 validation images: 16.6 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [13/100], Step [625/625], Loss: 3.3952\n",
      "Accuracy of the network on the 5000 validation images: 16.32 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [14/100], Step [625/625], Loss: 3.3752\n",
      "Accuracy of the network on the 5000 validation images: 19.25 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [15/100], Step [625/625], Loss: 3.3625\n",
      "Accuracy of the network on the 5000 validation images: 17.75 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [16/100], Step [625/625], Loss: 3.3545\n",
      "Accuracy of the network on the 5000 validation images: 17.8 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [17/100], Step [625/625], Loss: 3.3477\n",
      "Accuracy of the network on the 5000 validation images: 17.33 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [18/100], Step [625/625], Loss: 3.3397\n",
      "Accuracy of the network on the 5000 validation images: 19.74 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [19/100], Step [625/625], Loss: 3.3315\n",
      "Accuracy of the network on the 5000 validation images: 18.07 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [20/100], Step [625/625], Loss: 3.3252\n",
      "Accuracy of the network on the 5000 validation images: 18.69 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [21/100], Step [625/625], Loss: 3.3263\n",
      "Accuracy of the network on the 5000 validation images: 18.37 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [22/100], Step [625/625], Loss: 3.3112\n",
      "Accuracy of the network on the 5000 validation images: 19.51 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [23/100], Step [625/625], Loss: 3.3080\n",
      "Accuracy of the network on the 5000 validation images: 19.01 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [24/100], Step [625/625], Loss: 3.3170\n",
      "Accuracy of the network on the 5000 validation images: 18.14 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [25/100], Step [625/625], Loss: 3.3032\n",
      "Accuracy of the network on the 5000 validation images: 19.58 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [26/100], Step [625/625], Loss: 3.3034\n",
      "Accuracy of the network on the 5000 validation images: 20.15 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [27/100], Step [625/625], Loss: 3.3084\n",
      "Accuracy of the network on the 5000 validation images: 19.51 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [28/100], Step [625/625], Loss: 3.3043\n",
      "Accuracy of the network on the 5000 validation images: 20.69 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [29/100], Step [625/625], Loss: 3.2948\n",
      "Accuracy of the network on the 5000 validation images: 18.65 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [30/100], Step [625/625], Loss: 3.3017\n",
      "Accuracy of the network on the 5000 validation images: 19.19 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [31/100], Step [625/625], Loss: 3.3079\n",
      "Accuracy of the network on the 5000 validation images: 18.13 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [32/100], Step [625/625], Loss: 3.3009\n",
      "Accuracy of the network on the 5000 validation images: 17.66 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [33/100], Step [625/625], Loss: 3.2841\n",
      "Accuracy of the network on the 5000 validation images: 19.41 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [34/100], Step [625/625], Loss: 3.2970\n",
      "Accuracy of the network on the 5000 validation images: 19.42 %\n",
      "Current learning rate: 0.05\n",
      "Epoch [35/100], Step [625/625], Loss: 3.2793\n",
      "Accuracy of the network on the 5000 validation images: 19.02 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [36/100], Step [625/625], Loss: 2.6619\n",
      "Accuracy of the network on the 5000 validation images: 33.73 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [37/100], Step [625/625], Loss: 2.4815\n",
      "Accuracy of the network on the 5000 validation images: 35.7 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [38/100], Step [625/625], Loss: 2.4190\n",
      "Accuracy of the network on the 5000 validation images: 35.84 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [39/100], Step [625/625], Loss: 2.3863\n",
      "Accuracy of the network on the 5000 validation images: 36.83 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [40/100], Step [625/625], Loss: 2.3482\n",
      "Accuracy of the network on the 5000 validation images: 37.48 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [41/100], Step [625/625], Loss: 2.3235\n",
      "Accuracy of the network on the 5000 validation images: 36.99 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [42/100], Step [625/625], Loss: 2.2914\n",
      "Accuracy of the network on the 5000 validation images: 38.19 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [43/100], Step [625/625], Loss: 2.2525\n",
      "Accuracy of the network on the 5000 validation images: 37.61 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [44/100], Step [625/625], Loss: 2.2222\n",
      "Accuracy of the network on the 5000 validation images: 39.9 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [45/100], Step [625/625], Loss: 2.2077\n",
      "Accuracy of the network on the 5000 validation images: 39.67 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [46/100], Step [625/625], Loss: 2.1805\n",
      "Accuracy of the network on the 5000 validation images: 40.18 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [47/100], Step [625/625], Loss: 2.1566\n",
      "Accuracy of the network on the 5000 validation images: 41.35 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [48/100], Step [625/625], Loss: 2.1309\n",
      "Accuracy of the network on the 5000 validation images: 40.51 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [49/100], Step [625/625], Loss: 2.1206\n",
      "Accuracy of the network on the 5000 validation images: 41.0 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [50/100], Step [625/625], Loss: 2.1045\n",
      "Accuracy of the network on the 5000 validation images: 42.42 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [51/100], Step [625/625], Loss: 2.0821\n",
      "Accuracy of the network on the 5000 validation images: 42.5 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [52/100], Step [625/625], Loss: 2.0592\n",
      "Accuracy of the network on the 5000 validation images: 42.06 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [53/100], Step [625/625], Loss: 2.0494\n",
      "Accuracy of the network on the 5000 validation images: 42.35 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [54/100], Step [625/625], Loss: 2.0321\n",
      "Accuracy of the network on the 5000 validation images: 43.42 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [55/100], Step [625/625], Loss: 2.0203\n",
      "Accuracy of the network on the 5000 validation images: 43.63 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [56/100], Step [625/625], Loss: 2.0112\n",
      "Accuracy of the network on the 5000 validation images: 43.26 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [57/100], Step [625/625], Loss: 1.9872\n",
      "Accuracy of the network on the 5000 validation images: 43.68 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [58/100], Step [625/625], Loss: 1.9758\n",
      "Accuracy of the network on the 5000 validation images: 42.53 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [59/100], Step [625/625], Loss: 1.9647\n",
      "Accuracy of the network on the 5000 validation images: 45.23 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [60/100], Step [625/625], Loss: 1.9644\n",
      "Accuracy of the network on the 5000 validation images: 45.13 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [61/100], Step [625/625], Loss: 1.9433\n",
      "Accuracy of the network on the 5000 validation images: 43.85 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [62/100], Step [625/625], Loss: 1.9429\n",
      "Accuracy of the network on the 5000 validation images: 43.98 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [63/100], Step [625/625], Loss: 1.9306\n",
      "Accuracy of the network on the 5000 validation images: 43.92 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [64/100], Step [625/625], Loss: 1.9306\n",
      "Accuracy of the network on the 5000 validation images: 44.63 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [65/100], Step [625/625], Loss: 1.9046\n",
      "Accuracy of the network on the 5000 validation images: 44.6 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [66/100], Step [625/625], Loss: 1.9014\n",
      "Accuracy of the network on the 5000 validation images: 45.26 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [67/100], Step [625/625], Loss: 1.8925\n",
      "Accuracy of the network on the 5000 validation images: 43.89 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [68/100], Step [625/625], Loss: 1.8847\n",
      "Accuracy of the network on the 5000 validation images: 45.32 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [69/100], Step [625/625], Loss: 1.8820\n",
      "Accuracy of the network on the 5000 validation images: 45.08 %\n",
      "Current learning rate: 0.005000000000000001\n",
      "Epoch [70/100], Step [625/625], Loss: 1.8701\n",
      "Accuracy of the network on the 5000 validation images: 43.72 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [71/100], Step [625/625], Loss: 1.4714\n",
      "Accuracy of the network on the 5000 validation images: 54.88 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [72/100], Step [625/625], Loss: 1.3411\n",
      "Accuracy of the network on the 5000 validation images: 56.56 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [73/100], Step [625/625], Loss: 1.2807\n",
      "Accuracy of the network on the 5000 validation images: 56.51 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [74/100], Step [625/625], Loss: 1.2441\n",
      "Accuracy of the network on the 5000 validation images: 56.85 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [75/100], Step [625/625], Loss: 1.2149\n",
      "Accuracy of the network on the 5000 validation images: 57.4 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [76/100], Step [625/625], Loss: 1.1835\n",
      "Accuracy of the network on the 5000 validation images: 57.54 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [77/100], Step [625/625], Loss: 1.1675\n",
      "Accuracy of the network on the 5000 validation images: 57.67 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [78/100], Step [625/625], Loss: 1.1436\n",
      "Accuracy of the network on the 5000 validation images: 57.5 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [79/100], Step [625/625], Loss: 1.1180\n",
      "Accuracy of the network on the 5000 validation images: 57.72 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [80/100], Step [625/625], Loss: 1.1058\n",
      "Accuracy of the network on the 5000 validation images: 58.2 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [81/100], Step [625/625], Loss: 1.0874\n",
      "Accuracy of the network on the 5000 validation images: 58.29 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [82/100], Step [625/625], Loss: 1.0663\n",
      "Accuracy of the network on the 5000 validation images: 57.12 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [83/100], Step [625/625], Loss: 1.0563\n",
      "Accuracy of the network on the 5000 validation images: 57.62 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [84/100], Step [625/625], Loss: 1.0379\n",
      "Accuracy of the network on the 5000 validation images: 57.71 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [85/100], Step [625/625], Loss: 1.0358\n",
      "Accuracy of the network on the 5000 validation images: 58.05 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [86/100], Step [625/625], Loss: 1.0177\n",
      "Accuracy of the network on the 5000 validation images: 57.18 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [87/100], Step [625/625], Loss: 1.0027\n",
      "Accuracy of the network on the 5000 validation images: 57.71 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [88/100], Step [625/625], Loss: 0.9928\n",
      "Accuracy of the network on the 5000 validation images: 57.89 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [89/100], Step [625/625], Loss: 0.9848\n",
      "Accuracy of the network on the 5000 validation images: 58.23 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [90/100], Step [625/625], Loss: 0.9773\n",
      "Accuracy of the network on the 5000 validation images: 57.51 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [91/100], Step [625/625], Loss: 0.9625\n",
      "Accuracy of the network on the 5000 validation images: 57.59 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [92/100], Step [625/625], Loss: 0.9517\n",
      "Accuracy of the network on the 5000 validation images: 57.59 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [93/100], Step [625/625], Loss: 0.9508\n",
      "Accuracy of the network on the 5000 validation images: 57.45 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [94/100], Step [625/625], Loss: 0.9364\n",
      "Accuracy of the network on the 5000 validation images: 57.59 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [95/100], Step [625/625], Loss: 0.9258\n",
      "Accuracy of the network on the 5000 validation images: 57.31 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [96/100], Step [625/625], Loss: 0.9308\n",
      "Accuracy of the network on the 5000 validation images: 56.9 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [97/100], Step [625/625], Loss: 0.9197\n",
      "Accuracy of the network on the 5000 validation images: 56.9 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [98/100], Step [625/625], Loss: 0.9023\n",
      "Accuracy of the network on the 5000 validation images: 57.15 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [99/100], Step [625/625], Loss: 0.8962\n",
      "Accuracy of the network on the 5000 validation images: 56.51 %\n",
      "Current learning rate: 0.0005000000000000001\n",
      "Epoch [100/100], Step [625/625], Loss: 0.8806\n",
      "Accuracy of the network on the 5000 validation images: 57.84 %\n",
      "Current learning rate: 0.0005000000000000001\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hVVb7/8fc6Jb1XQgpJ6EhCCAEUEEHsIiIKig10xvpTdLyWcYr1emfG4VqYUedawMaIFURRURgRCwqEaoAQSiCNNEjv56zfH/vQQ0hCTk5y8n09Tx6Sfc7e+7sCfLKy9tprK601Qggh3I/J1QUIIYRwDgl4IYRwUxLwQgjhpiTghRDCTUnACyGEm7K4uoBjhYWF6fj4eFeXIYQQ3UZ6enqJ1jq8ude6VMDHx8ezfv16V5chhBDdhlJq36lekyEaIYRwUxLwQgjhpiTghRDCTXWpMXghROdobGwkNzeXuro6V5ciWsnLy4uYmBisVmur95GAF6IHys3Nxd/fn/j4eJRSri5HnIbWmtLSUnJzc0lISGj1fjJEI0QPVFdXR2hoqIR7N6GUIjQ0tM2/cUnAC9FDSbh3L+35++r2Aa+1Zt7KLL7bWezqUoQQokvp9gGvlOK11Xv4LlMCXojuorS0lJSUFFJSUujVqxfR0dFHvm5oaGhx3/Xr1zNnzpw2nS8+Pp6SkpIzKblbcouLrAHeVspqW/5HIYToOkJDQ9m0aRMATzzxBH5+fjz44INHXm9qasJiaT6e0tLSSEtL65Q6u7tu34MHCPKxUl7T6OoyhBBnYPbs2TzwwANMnDiRRx55hLVr1zJmzBiGDx/OmDFjyMzMBGDVqlVMnjwZMH443HrrrUyYMIHExETmzZvX6vPt27ePSZMmkZyczKRJk9i/fz8AH374IUOHDmXYsGGMHz8egIyMDEaNGkVKSgrJyclkZWV1cOudwy168EE+VspqJeCFaI8nP8tgW35Fhx5zSO8AHr/irDbvt3PnTlasWIHZbKaiooLVq1djsVhYsWIFf/jDH/j4449P2mfHjh18++23VFZWMnDgQO66665WzRW/5557uPnmm5k1axbz589nzpw5LFmyhKeeeorly5cTHR1NWVkZAP/617+47777uOGGG2hoaMBms7W5ba7gHgHv7cGO8o79ByqE6HzTp0/HbDYDUF5ezqxZs8jKykIpRWNj8524yy+/HE9PTzw9PYmIiKCwsJCYmJjTnmvNmjV88sknANx00008/PDDAIwdO5bZs2czY8YMpk2bBsA555zDM888Q25uLtOmTaN///4d0Vync2rAK6WygUrABjRprZ0ycBboY6VcevBCtEt7etrO4uvre+TzP//5z0ycOJHFixeTnZ3NhAkTmt3H09PzyOdms5mmpqZ2nfvwNMR//etf/PLLLyxbtoyUlBQ2bdrE9ddfz+jRo1m2bBkXX3wxr7/+Oueff367ztOZOmMMfqLWOsVZ4Q4Q5G2lrKYRrbWzTiGE6GTl5eVER0cD8Oabb3b48ceMGcOiRYsAWLhwIePGjQNg9+7djB49mqeeeoqwsDBycnLYs2cPiYmJzJkzhylTprBly5YOr8cZ3OYia5NdU93QPcbFhBCn9/DDD/Poo48yduzYDhnzTk5OJiYmhpiYGB544AHmzZvHggULSE5O5p133uHFF18E4KGHHiIpKYmhQ4cyfvx4hg0bxvvvv8/QoUNJSUlhx44d3HzzzWdcT2dQzuz1KqX2AocADfyf1vrVlt6flpam2/PAjw/W5fDwx1v44ZGJxAT7tK9YIXqQ7du3M3jwYFeXIdqoub83pVT6qUZInH2RdazWOl8pFQF8o5TaobVefUJxtwO3A8TFxbXrJIE+xhXzsppGYoLPrGAhhHAXTh2i0VrnO/4sAhYDo5p5z6ta6zStdVp4eLOPFTytIG8j4CvkQqsQQhzhtIBXSvkqpfwPfw5cBPzqjHMF+XgAyFx4IYQ4hjOHaCKBxY6pRxbg31rrr5xxokDvo0M0QgghDE4LeK31HmCYs45/rKDDY/CyHo0QQhzhFtMkvaxmPC0mWY9GCCGO4RYBD471aCTghegWJkyYwPLly4/b9sILL3D33Xe3uM/hadSXXXbZkXVijvXEE08wd+7cFs+9ZMkStm3bduTrxx57jBUrVrSl/GYduwhaV+E+Ae/tIUM0QnQTM2fOPHIX6WGLFi1i5syZrdr/iy++ICgoqF3nPjHgn3rqKS644IJ2Haurc5uAD5QevBDdxjXXXMPnn39OfX09ANnZ2eTn5zNu3Djuuusu0tLSOOuss3j88ceb3f/YB3g888wzDBw4kAsuuODIksIAr732GiNHjmTYsGFcffXV1NTU8NNPP7F06VIeeughUlJS2L17N7Nnz+ajjz4CYOXKlQwfPpykpCRuvfXWI/XFx8fz+OOPk5qaSlJSEjt27Gh1W997770jd8Y+8sgjANhsNmbPns3QoUNJSkri+eefB2DevHkMGTKE5ORkrrvuujZ+V0/mFqtJgjEXfv/BGleXIUT38+Xv4cDWjj1mryS49K+nfDk0NJRRo0bx1VdfceWVV7Jo0SKuvfZalFI888wzhISEYLPZmDRpElu2bCE5ObnZ46Snp7No0SI2btxIU1MTqampjBgxAoBp06Zx2223AfCnP/2JN954g3vvvZcpU6YwefJkrrnmmuOOVVdXx+zZs1m5ciUDBgzg5ptv5pVXXuH+++8HICwsjA0bNvDyyy8zd+5cXn/99dN+G/Lz83nkkUdIT08nODiYiy66iCVLlhAbG0teXh6//mrMHD883PTXv/6VvXv34unp2ewQVFu5TQ9exuCF6F6OHaY5dnjmgw8+IDU1leHDh5ORkXHccMqJvv/+e6666ip8fHwICAhgypQpR1779ddfOffcc0lKSmLhwoVkZGS0WE9mZiYJCQkMGDAAgFmzZrF69dEb7w8vHTxixAiys7Nb1cZ169YxYcIEwsPDsVgs3HDDDaxevZrExET27NnDvffey1dffUVAQABgrJdzww038O67757yiVZt4T49eB8PWTJYiPZooaftTFOnTuWBBx5gw4YN1NbWkpqayt69e5k7dy7r1q0jODiY2bNnU1dX1+JxDi/ze6LZs2ezZMkShg0bxptvvsmqVataPM7p1uU6vCxxW5YkPtUxg4OD2bx5M8uXL+ell17igw8+YP78+SxbtozVq1ezdOlSnn76aTIyMs4o6N2mBx/obaW20UZdo6woKUR34Ofnx4QJE7j11luP9N4rKirw9fUlMDCQwsJCvvzyyxaPMX78eBYvXkxtbS2VlZV89tlnR16rrKwkKiqKxsZGFi5ceGS7v78/lZWVJx1r0KBBZGdns2vXLgDeeecdzjvvvDNq4+jRo/nuu+8oKSnBZrPx3nvvcd5551FSUoLdbufqq6/m6aefZsOGDdjtdnJycpg4cSLPPvssZWVlVFVVndH53agHf3Q9Gi+r2cXVCCFaY+bMmUybNu3IUM2wYcMYPnw4Z511FomJiYwdO7bF/VNTU7n22mtJSUmhT58+nHvuuUdee/rppxk9ejR9+vQhKSnpSKhfd9113HbbbcybN+/IxVUALy8vFixYwPTp02lqamLkyJHceeedbWrPypUrj3ua1Icffshf/vIXJk6ciNaayy67jCuvvJLNmzdzyy23YLfbAfjLX/6CzWbjxhtvpLy8HK01v/vd79o9U+gwpy4X3FbtXS4Y4PMt+dzz7418/bvxDIj07+DKhHAvslxw99TW5YLdZogmyNux4JhcaBVCCMCdAv7ImvBys5MQQoAbBfyRFSVlJo0QrdKVhmfF6bXn78ttAv5wD14WHBPi9Ly8vCgtLZWQ7ya01pSWluLl5dWm/dxmFo2fpwWzScl6NEK0QkxMDLm5uRQXF7u6FNFKXl5ex83QaQ23CXilFEHecjerEK1htVpJSEhwdRnCydxmiAaMBcfkblYhhDC4VcAHeUvACyHEYe4V8D4eMkQjhBAO7hXw3la5yCqEEA5uFfABcpFVCCGO6P6zaGxNsO8H8I0gyMdKZV0TTTY7FrNb/ewSQog26/4pqBS8dz2kv0mQ427WirrWrdUshBDurPsHvMkMvVMgfwNBPocXHJNxeCGE6P4BD9B7OBRsIcjTuO1a1qMRQgh3CfjoVLDV06tuLyDr0QghBLhNwBtPUQ+rMJ5QLjc7CSGEuwR8UB/wDsG/dAsgY/BCCAHuEvBKQfQIPAo3ATIGL4QQ4C4BDxCdiireQaRXk9zsJIQQuFPA904FbWecXz47CytdXY0QQric+wR8dCoAl4XksXbvQSrrpBcvhOjZ3Cfg/SIgMJYU016a7JrVO0tcXZEQQriU+wQ8QHQqIWVbCfKxsnJHoaurEUIIl3KvgO+diirbx+WJVlZlFmOzywOFhRA9l3sFvGMc/oqIIg5WN7Ap55CLCxJCCNdxr4CPSgEUKWoXFpNixfYiV1ckhBAu4/SAV0qZlVIblVKfO/tceAVAdCpe2z5kdJ9A/iMBL4TowTqjB38fsL0TzmM490E4tJc7gn4hs7CSnIM1nXZqIYToSpwa8EqpGOBy4HVnnuc4Ay+F3qmck/sGVpr4zw7pxQsheiZn9+BfAB4G7E4+z1FKwcQ/Yq3M5e7ANSxal0OTrfNOL4QQXYXTAl4pNRko0lqnn+Z9tyul1iul1hcXF3fMyftNgphR3GlazJ6CEt78KbtjjiuEEN2IM3vwY4EpSqlsYBFwvlLq3RPfpLV+VWudprVOCw8P75gzKwXn/xHv2gM81nsdz32zk7yy2o45thBCdBNOC3it9aNa6xitdTxwHfAfrfWNzjrfSRLOg7hzuLbpM5S28/inv6K13PgkhOg53Gse/LGUglG3Y6nYz9zhxazYXsTyDFm+QAjRc3RKwGutV2mtJ3fGuY4z+Arwi+Ti2s8ZHBXAE0szqKpv6vQyhBDCFdy3Bw9gtsKI2ZiyvuHZSYEcqKhj3sosV1clhBCdwr0DHiB1FigTSQUfc21aLPN/2EvmAXkgiBDC/bl/wAdGw6DLYMPbPHJhPH5eFv68RC64CiHcn/sHPMDI30LtQUKyv+CRSwaxNvsgn2zIc3VVQgjhVD0j4BPOg9D+sOIJro3IZXhcEI8vzWB99kFXVyaEEE7TMwJeKZi+ACxemN6azNv9VtPLz8LN89fy855SV1cnhBBO0TMCHqBXEtyxGs6aiv9Pf+WL0OdJCqhh9oK1/LhLnt8qhHA/PSfgwVgv/uo34Ip5eBSk817TA9zgv4lbFqzjg/U5rq5OCCE6VM8KeDCGa0bMgju+xxQSz59r/sqCwFd58aOVPP35Nll5UgjhNnpewB8W1g9+8w2Mf4gx9T+w2vu/iPv5Me57/UsKymVhMiFE99dzAx6MO13P/xNqzkbMqTdyk/Vb/pb/G5567kXeX7df5soLIbq1nh3whwXGwBUvYrp3HR7hibyk/sa2JXO5ef5adhdXubo6IYRoFwn4Y4Uk4nHb16gBF/Ok9S0u3z+XK543xubLaxtdXZ0QQrSJBPyJPP1Q1y2EMfdynfqalQFPsOanVUz631Wk7zvk6uqEEKLVJOCbYzLDRf8N139AlLmKz70e4za1lBteX8O3mfIQbyFE9yAB35IBF8PdP2MadBl3NL7NXJ93ue2tdSzemOvqyoQQ4rQsri6gy/MNhelvwTePMfmneXgGK25/386OA5U8eNFArGb5GSmE6Jok4FtDKbjwKVCKC398kfejzVz73XTWZx/iHzOH0zvI29UVCiHESaT72VpKwQVPwrjfMar0Uz4fsYkdBRVcNu97Vsm4vBCiC5KAbwulYNLjMORKztr2PN9M96RXgBe3vLmOF1bsxG6XG6OEEF2HBHxbKQVT/gFBcfT++m4Wzx7EVcOjeWFFFnfPX0VGXpncASuE6BJUVwqjtLQ0vX79eleX0Tr5m+CNCyHubHSvZMq3fklQ1W7ydCjrrSNp6HsRfcdMJSU2BJNJNX+M8jyoKoSQBPAO7tz6O4qtCZpqwdPf1ZUI0SMppdK11mnNviYBfwbWvgZfPAhmD+gzhppeoyjdnU540Y946Tr+2XQlb3rdxMSBEYxMCGFIVAADIv3xsJggawV8cBM01hjH8gqCs6bCpX8Hi8fRc+xaCRX5kHqTa9rYkqYGeOsKqDoAd/8MVrnYLERnayngZRbNmRj5W4g7G4ITwNMPH8AHoKmeho/v5O4dSzkYO52PMg7wYboxd95qVsz2W8sj9fMo9EpkY9/fEE0JveqyiEp/k6bi3ZhnvovyDIDVf4dV/2Ocq7EGRt/hqpY2b8XjkPOz8fnaV2Hsfa6tRwhxHOnBO0tlIfxjBPQ5B9vMD9hXWk1GXjk+G19l0r4X2GpN5l79IPuqLRz+K5hmWs3frK+xhxhKrZGMaVrLpuBLCDDVklC6moxzX6Kx/2XYtabJpvG0mhnUyx8vq7n5Gooz4ccX4byHITi+7W3I22AMHYUknPzatk/hg5th9J1wcA/k/AJzNoFPSNvPI4RoNxmicZWf/gFf/wmu/wASJ8Cy/4KN78DgK2Da62D1otFmp7iyngMVdRwor0Pt/g/nb3kQi72O17x/yyu1F1BfV82/PZ5hiNrHPY1zsKNIVAX4UM8aNQxb1HCGxgQT6G3Fx8OCxaSw5PzEjN2P4GuvotAjlq/PeZcBfWLx8bCg0dg1VNQ2cqimgYraRqxmEwHeVgK8rPQK9CLWXIrnK6PAZMF+5csc7HMJxZX1FFfWU3NgJ5NWz6A2sB9F13xCrC0Pz9fHw5h7jCUehBCdRgLeVZoa4JUxYG8C33DIXQvjH4IJfwBTCxOYSndDQzVEJQNQ09BEaVE+4e9fgVdl9klvLzMF861tGOlNCWTY4+mjCnnW+ioHzL1YFnQDvzn4v2y09+OmhkdpwEoEh7jCvIZ6rOTpMHJ1OFk6Gjh6MfhF6z+5xLyePSqOwXoX/2iayre2FGaYV3GFeQ0NWLm8/n/IJwyrWfGa/xuMq1/NZ+d9RnqZL5m5JUTU7OTiwDxSzbsJ9zXhOWMBmGVUUIiOJAHvSrtWwrvTwOoDU18xLqS2V1UR7PkOgvtAaD9jymbWCshcBru/hbqyI2/Vfcagrvu3McSy9SP4+DeUxFxAk/IgIvdrTLrpuEPXDJjK/gnPU16nqdnzMxN/uJ7loTexIvxmbjz4T4YVfQqAzeJDzYArKR92OzmWPhRW1LH9QAW7snbwcult7NK9UcrEALUfCzbj2NoTH1VP7d0b8Y5IbH/7hRAnkYB3ta0fQeRQiBjkvHNoDeU5ULAFag9C0gyweh19/fvnYOWT4BlozMhJuxU8fKEsB3Z+Bd/PheTr4MqXYP7FUJ4L96aDp59x7O2fQV258QPqFFMi61f+Bcu6/8MUNQwVnQq9h9MUlcrKVSu5ePMc9k9bSlzyec77HgjRA8ksGldLusb551AKguKMj+aM+x3Ej4OIIUZoH+bfC2JHGlMc//M0lGRC/ka48uWj71MKhkw5bQmekx6FSY8et80CxMT2gc1QUZLfzsYJIdpDAr6nUApiR5369fEPgq0RvvsrRA2DYTM77NTB4dEA1B0q6LBjCiFOr1UBr5TyBWq11nal1ABgEPCl1lqeY+dOJvwewgdC7+EtXwRuo7DIGAAayws77JhCiNNr7f/i1YCXUioaWAncArzprKKEiygFQ6c1P+/9DHh4eVOBL1TLqptCdKbWBrzSWtcA04B/aK2vAoY4ryzhbspNwVhqi11dhhA9SqsDXil1DnADsMyxTcbvRavVeITgVX/Q1WUI0aO0NuDvBx4FFmutM5RSicC3zitLuJsGrzD8bRLwQnSmVvXCtdbfAd8BKKVMQInWeo4zCxPuRfuGE3LoZ+oabadeO0cI0aFa1YNXSv1bKRXgmE2zDchUSj10mn28lFJrlVKblVIZSqknO6Jg0T2Z/CIJUDUUlpad/s1CiA7R2iGaIVrrCmAq8AUQB5xugfJ64Hyt9TAgBbhEKXV2uysV3ZpXUC8ASoryXFyJED1HawPeqpSyYgT8p4757y2ucaANVYf3d3x0nXURRKfyDY0CoKJEAl6IztLagP8/IBvwBVYrpfoAFafbSSllVkptAoqAb7TWvzTzntuVUuuVUuuLi2UanbsKctzNWntQ7mYVorO0KuC11vO01tFa68scPfN9wMRW7GfTWqcAMcAopdTQZt7zqtY6TWudFh4e3uYGiO7BO9jowTeWH3BxJUL0HK29yBqolHrucE9bKfW/GL35VtFalwGrgEvaV6bo9nwjANBV8luaEJ2ltUM084FKYIbjowJY0NIOSqlwpVSQ43Nv4AJgR/tLFd2a1Ysa5YNZ7mYVotO09m7Uvlrrq4/5+knH2HpLooC3lFJmjB8kH2itP29PkcI9VFtD8KovdXUZQvQYrQ34WqXUOK31DwBKqbFAbUs7aK23AMPPsD7hRuq9wvGvO0SjzY7V3HGrVQohmtfagL8TeFspFej4+hAwyzklCXdl9wkjrCyDosp6ooO8XV2OEG6vtbNoNjtuWEoGkrXWw4HznVqZcDsm/0jCVDkHylv85U8I0UHa9Huy1rrCcUcrwANOqEe4Mc+gSIJUNUUHK11dihA9wpkMhKoOq0L0CL7BvQEok7tZhegUZxLwsuyAaBPvYGM9mhq5m1WITtHiRValVCXNB7kC5CqZaBPlFwlAvdzNKkSnaDHgtdb+nVWI6AH8HEtRVMmzWYXoDDIZWXQex3IFppoSFxciRM8gAS86j4cP9WYfPOtLsNvlEo4QziYBLzpVvWcYoZRTUlXv6lKEcHsS8KJTKb8Iwijn+ywZphHC2STgRafyC40iylLJxxtyXV2KEG5PAl50KuUbQS9zBWv2lJJXJksWCOFMEvCic/lF4N1Ujlk3sVh68UI4lQS86Fy+xlz4i2PtfLwhD61lNo0QziIBLzpX/DgwWXnE8h57S6rYsL/M1RUJ4bYk4EXnCh8IEx8lrmA513iskYutQjiRBLzofGPvh9jRPG1ZwLrNW6hrtLm6IiHckgS86HwmM1z1LzxMmmfsL/LVP+Zge+9GmH8J7P6Pq6sTwm1IwAvXCEnEfOlfGWXK5IryhRTtSsdelgsLp8OGt11dnRBuobXPZBWi442YBYnn8c0+zb0fbifZ38xbvf+J39J74eBemPB7sHi6ukohui3VlaappaWl6fXr17u6DOEC3+0s5p6FG6itr2NB+PucW7kMzJ4QPQLizoak6RA5xNVlCtHlKKXStdZpzb4mAS+6irKaBub/mM2CH/cwoiGdmyKzOdcjC4/irWBvgsSJcM7/g76TwCSji0KABLzoZirqGnn9+728uno3dg33nh3Cb72/w3vjG1B1wLhZqt+FMOAiUGYo2g4lmdDvAki53tXlC9GpJOBFt1RQXsuzX2WyeGMeVrPiwoHB3BGxnaTKHzDtXgl1h2+SUuATAjWlMOUfkHqzS+sWojNJwItuLfNAJR+uz2HJpnxKquqJDvLmrnPjmB5dgqeHJ4QNBJMF3rsO9nwL09+EIVe6umwhOoUEvHALTTY732YW88qqXWzYX0a4vyf3TOzHDaPjsJhN0FANb0+Fgk0w6TEI6A2eARDWH4LjXV2+EE4hAS/citaan/cc5MWVO/l5z0EGRPrx+BVnMbZfGNQegremwIEtx+/U70IYfcepL9DamsAss4ZF9yMBL9yS1pqvtxXy38u2kXOwlvMGhHPH+ETOSQhEVRVCfRXUVxh3x66fD1WFEJkEM96C0L7GQRrr4LM5kPkVzHgT+p7v0jYJ0VYS8MKt1TXaWPBjNm/8sIeSqgbO6h3AjLRYUuOCGRTlj9VsgqYGyFgMXz0Cdjtc/RpEpcCi6yFvPQTEGDN0pr4CyTNc3SQhWk0CXvQIdY02lmzM4/Uf9rKrqAoAT4uJsxNDmTkqjgsGR2CpyIH3b4QDW42ZN421cNX/QeJ5sOgGyP4eJj0OY+8z1swRoouTgBc9itaavLJaNu4vY+P+Mr78tYCC8joiAzyZkRbLVUkhJP7yBOT8DNfMh6hhxo5N9bD4Tsj4BMIGwPiHYei0k4Nea+NCbuUBqCuHpjoYPMX4gSFEJ5OAFz3a4dk37/68j++zirFrSI4JZGpKNFOHRxPi63H0zXY7bP8UvnsWirZBSF9IvtaYdhnWH7Z9Cj88f/JF3F5JMOtz8A5qW3F15WC3yQ8H0W4S8EI4FFbU8dnmfD7ZkMe2ggo8zCYuHBLJlJTeDO4VQHSwN2aTMoJ+x2fw8yuw/2dAG1Mu6ysgtB+MudcIda8gKN4BH8yC3sPhpsXg6Qf71hg/CLwCYOjVxsXbExdO27MKPv6tMYf/tyshMNoV3xLRzUnAC9GMbfkVfJiew5KNeRyqaQTAw2Kib7gfI/oEMSohlNEJIUSqMtj+GeSug0GTYdDlJw/bbFsKH86CPmPBww92fgm+EWBvNKZuegbCwEtgwMXGmjrrXodv/8f4raCiAIL7wC1fGj8QhGgDCXghWlDfZGNzTjl7iqvYU1LN9oIKNuw7RHWD8aSplNggJidHcWlSFNFB3qc+0Kb3YMmdRpiPux9G3wlmq9FT//UTyFpuLKdwWNIMmPy8cS1g4QzoOxFmvn/8fHxbE2z9AAozjNk9h68XNNbCpn/D/jUwYrbxrNuOoLUxNOUXCb5hHXNM4VQS8EK0UZPNzraCCr7PKuGLrQVk5FcAMKJPMJOTo7g8KYqIAK+TdyzYDIGxzY+p222Qlw67VhjDPEnTQSnjtfQ34bP7jBux+l0AEYOhPBe+nwuHskGZQNsh7hyIHQ0b34WaErD6QGON8ZvFuAeM4aKdX0LOOogdBUnXQP+LoTIf9nwHOb8Y4R2TZizFbPEyhp1qD0HWCtjyPpRmwZCpxv0CostzScArpWKBt4FegB14VWv9Ykv7SMCLriq7pJplWwv4bHM+Ow5UAhAV6EVCmC8JYb6kxgUzpl8oUYEt9PBP5/vn4OeXobr46LaoYTDhUWNN/I0LYe2rULbPCO2xc4yQXvOSMd7fYEwNxb+3Ee77fjSOZbIaQ0UAPmGOC7uNzdfQZyzUHDR+mNyztv1tEZ3GVQEfBURprTcopfyBdI1ikUkAABI2SURBVGCq1nrbqfaRgBfdwa6iSr7eVsiuoir2llSzu6iKiromABLCfEmOCWRQrwAG9fInMdyX3kHexs1WrVVVbAyTKJMx9HK4lw/GbwG1ZeAbesI+RZD5hXHzVtQwYx9bE2SvhqxvICQREs4zxvyb6o1ZQPkbjeN5BRgXkHunQFAcfPOYcXH5jwfkXoBuoEsM0SilPgX+qbX+5lTvkYAX3ZHdrsksrOSn3aWs2V3Ktvxy8svrjrxuNil6B3kxMj6EqSnRjOkbaiyO1lWlv2Us33DfZlmkrRtoKeA7ZXUlpVQ8MBz4pZnXbgduB4iLi+uMcoToUCaTYnBUAIOjAvjNuAQAymsb2VlYyd6SavaX1rC3tJpvthXyyYY8wvw8GT8gjEG9/BkQaXxEBXqhju2pu1JoP+PP0t0S8N2c0wNeKeUHfAzcr7WuOPF1rfWrwKtg9OCdXY8QnSHQ28rI+BBGxh+92FrXaGNVZhFLNubzQ1YJn2zIO/Kan6eFfhF+9A7yQqFAQaivB5MGR3JOYigelk7s8R9eiK10N/Sb1HnnFR3OqQGvlLJihPtCrfUnzjyXEF2dl9XMJUOjuGRoFACHqhvILKwkq6iKrMJKdhZWklVYhcZYbqGgvI631+zD39NCWnwwHhYTCoW3h5n+kX4MiQqgX4QfHo7hHg+LiSAfjxYqaCW/SGMu/8HdZ34s4VJOC3hl/L75BrBda/2cs84jRHcV7OvB2YmhnJ0Y2uzrdY02ftxVwvKMA2zNq0BrjV1rquqaWLwxr9l9RsWHcO3IWC5LisLbo50XSJUyLsqWSsB3d86cRTMO+B7YijFNEuAPWusvTrWPXGQVonXKaxrZfqCCvSXV2B3/h0urGli8MY+9JdV4W81EBHji42HBz9NMVKA3cSE+xIX4EBPsTe8gb3oFeuFlPcUPgQ9nQ/4muG9T5zVKtItLLrJqrX8AushVIyHcS6CPtdne/73n92Pt3oN8+esBDtU0UF1vo7KukY05h1i2tQCb/fgOnZfVhLfVjLfVTO8gbwb28mdQVABX+cXjV7YUbI3G3biiW5JnlAnhRpRSjE4MZXQzwz6NNjsFZXXkldWS7/ioqm+ittFGdb2NnIM1LN2cz8Jf9tMU58Et2gaH9kFYPxe0RHQECXghegir2URcqA9xoT6nfI/WmrsXbmBtzn5uASjdJQHfjXXhuy2EEJ1NKcWgXgH8XBFsbJCZNN2aBLwQ4jgDIv04pP1p8gyUmTTdnAS8EOI4/SP9AajwjjOGaES3JQEvhDhOfKgPHmYT+eZoOLjH1eWIMyABL4Q4jsVsIjHclyxbJJTnGA8XEd2SBLwQ4iT9I/3ZXO2Yanlwr2uLEe0mAS+EOMmACD/WVzkWSpOZNN2WBLwQ4iT9I/3J1r2ML+RCa7clAS+EOMmASD+q8KHOI0SmSnZjcierEOIkfUJ98bCYyPfqT+LWD8E3DMb9DrwCXV1a92K3Q12Z8WDzunKorzIuWjdWG8/KDegNAdHG99cJD3yRgBdCnMRsUvQN92Oez/28EL/EeKh3+ltw7n9B2q3gcerlDtxWSRbsWGYEcuwo4/m1JVmw9UPYvtR4T1AfY3tdORRvN15vqmv5uAC+4fBQxw+FScALIZo1INKPddmNcNurcPbdsOJx+PqP8OOLMHYODLrcGL4p2gYNNZBwLsSMAksHPHTkWE0NsGmhEYIDLwNTK0aWi3bAoWxjH98w8O8FFs/m31tdChvehMyvjAedWzzA4m3sExgDVh8jwHNOeOKodzDUHgKU8XB0zwAo2w/7fwZPPwgfZDzoPDDG+M3HM8DYbvUFqzfY6qEi3/hoqj/Db1LzOu2h260h68EL0XW89O0u/r48k4wnL8bX09EXzP4Rvvsb7P3u+DcrE2i78SSoxAkwbCYMuPj4pYa1huoSKNlpBKHJYvwm4OFnPPs1MPbk8C7cBovvgANbjK8jzoLxDxq95JxfIGct2JuM58iG9YeKAsj4xPihcyyLt/H4wcFXQMxII1TL9sP+n2DrR0YvOzrNqMfWCA3VUFkA1cXG/mEDIOUGSJoONSXGefM3QeQQOGsaBER10He97Vz+0G0hRPfTP8IPgKyiKlJig4yN8WMhfqkRcIUZRi81YhAoM+xdDbtXGsMYOz43es/9LzbGoMv2Gx91Zac+odXHCOnAWOOxgSYLpC8wer4z3jFCePXf4aNbju4TGAdWL9i5HOyNxrbYs+HSZ6F3KtSUQnURFGw5WteJ5xw2E0bfARGDT66psQ5qD4J/1NEx8sBoiBrWzu9q55IevBCiWdkl1UyYu4pnr0lmRlps63e0NcGuFbDxHdj3oxHWgbFGrzusv/ERnGD0+BuqjQuQpbuNnn1xptFzrjxgDH8MngyXPw9+4cax7TbI+toY0ogdfbTnbGuCsn1GYJ+qN223Q/5Go3cfGGPUExjb8UNKnUx68EKINosN8cHTYiKrsLJtO5otMPAS46O1EsafvM1uA9MJjxQ0mWHgpc2fM7Rvy+cwmSBmhPHRQ8g8eCFEs8wmRb8IPzbnlLumgBPDXbSZBLwQ4pQmJ/dmbfZB0vcdcnUpoh0k4IUQpzRrTB/C/Dx47ptMV5ci2kECXghxSj4eFu6a0I8fd5WyZnepq8sRbSQBL4Ro0Q2j44gM8OS5bzLpSrPuxOlJwAshWuRlNXPPxH6syz7E6qwSV5cj2kACXghxWjNGxhId5M2Tn2VQWNGKtVVElyABL4Q4LU+LmbnTh1FYXse0l39id3GVq0sSrSABL4RolXP6hrLo9nOob7JxzSs/sWG/TJ3s6iTghRCtlhQTyEd3jsHfy8r0f63hj4u3UlQpQzZdlQS8EKJN4sN8+fT/jeXG0XG8vy6HCX9fxXNfZ1JS5Zwlb0X7yWJjQoh221tSzd+X7+CLrQfwMJuYktKbW8cmMKR3gKtL6zFaWmxMAl4IccZ2FVXx1k/ZfJSeS22jjfEDwrl7Ql9GJ4SgnPAoOnGUBLwQolOU1zSycO0+5v+wl5KqBpJjAjlvQDgj40NI7ROMn6csYNvRJOCFEJ2qrtHGh+m5fLg+h1/zyrFrsJgUEwaGc2VKNBcMjsTbQ1aL7AgS8EIIl6mqb2Lj/kOs3lnMZ5sLOFBRh5+nhcnJUcwYGcvw2CAZxjkDEvBCiC7BZtf8sreUTzbksWxLAbWNNvqG+zI6MZTBvfwZFBVAUnQgXlbp3beWBLwQosupqm/i8835LN2cT0Z+BeW1xjNVvawmzk4M5dz+4UwaFEF8mK+LK+3aJOCFEF2a1poDFXX8mlfBj7tKWJ1VzJ7iagAGRvpz0VmRjOgTTN9wP3oHeWM2yZDOYS55JqtSaj4wGSjSWg911nmEEN2fUoqoQG+iAr25cEgkADkHa/hmWyHLMw7w0re7sDv6oh4WE0N7BzB+QDjn9g9nYC9/vCwmLGa5b/NETuvBK6XGA1XA260NeOnBCyGaU1bTwM7CKvYUV7G7uIq12YfYklvGsfFlNimiAr04t38Y4/qFc07fUEJ8PVxXdCdxSQ9ea71aKRXvrOMLIXqOIB8PRiWEMCoh5Mi2Q9UN/LS7lLyyGuob7dQ12cgqrOLzzQW8tzYHgMQwX1L7BDMsJpDEcD8SwnzpFeCFqYcM8bj8rgOl1O3A7QBxcXEurkYI0V0E+3pweXLUSdubbHY255bxy96DbNh3iP/sKOKj9Nwjr3tYTEQHeRMd5E3vIC+iAo0/o4N8SIoOJNDH2pnNcCqnXmR19OA/lyEaIYSrHL6Au7ekmr0l1ewrrSHvUC25ZbXkl9VSUlV/3FBP33BfhscFMzDSn34RfiSG++LtYcasFFaLCX9PS5eat++SIRohhOgKjr2AO6Zv2EmvNzTZKayoY19pDZtyDrFhfxnfntDrP1aEvyfD44JIiQ1mUJQ//cL9iA7yRimoabBRVttIgJcFfy/X/yYgAS+E6NE8LCZiQ3yIDfFhXP+jPwAOVTewq7iKvSXVNDTZsdk1dY02thdUsDGnjOUZhccdAw0NNvuRbb0DvegX6U9ydCBj+oUyok8wnpbOvYHLmbNo3gMmAGFAIfC41vqNlvaRIRohRHdx+AfAriJjdo/JpAj28SDI20ppdQO7iqrIPFBJZmElNrvG02JiYC9/wv08CfPzJC7Uh7Q+wQyLDTqjO3ddNYtmprOOLYQQrhbs68FI3xBGxoe0+L6KukbW7jnIj7tL2F1cTUF5HVvzyimqNB6QYjUrUmKDeP/2czp8do8M0QghhBMFeFm5YEgkFzhu4DrsUHUD6fsOsX7fIcpqGpwydVMCXgghXCDY16PZ4O9Icm+vEEK4KQl4IYRwUxLwQgjhpiTghRDCTUnACyGEm5KAF0IINyUBL4QQbkoCXggh3FSXeiarUqoY2NeGXcKAEieV01X1xDZDz2x3T2wz9Mx2n0mb+2itw5t7oUsFfFsppdafapEdd9UT2ww9s909sc3QM9vtrDbLEI0QQrgpCXghhHBT3T3gX3V1AS7QE9sMPbPdPbHN0DPb7ZQ2d+sxeCGEEKfW3XvwQgghTkECXggh3FS3DHil1CVKqUyl1C6l1O9dXY+zKKVilVLfKqW2K6UylFL3ObaHKKW+UUplOf4MdnWtHU0pZVZKbVRKfe74uie0OUgp9ZFSaofj7/wcd2+3Uup3jn/bvyql3lNKebljm5VS85VSRUqpX4/Zdsp2KqUedeRbplLq4vaet9sFvFLKDLwEXAoMAWYqpYa4tiqnaQL+S2s9GDgb+H+Otv4eWKm17g+sdHztbu4Dth/zdU9o84vAV1rrQcAwjPa7bbuVUtHAHCBNaz0UMAPX4Z5tfhO45IRtzbbT8X/8OuAsxz4vO3KvzbpdwAOjgF1a6z1a6wZgEXCli2tyCq11gdZ6g+PzSoz/8NEY7X3L8ba3gKmuqdA5lFIxwOXA68dsdvc2BwDjgTcAtNYNWusy3LzdGI8N9VZKWQAfIB83bLPWejVw8ITNp2rnlcAirXW91novsAsj99qsOwZ8NJBzzNe5jm1uTSkVDwwHfgEitdYFYPwQACJcV5lTvAA8DNiP2ebubU4EioEFjqGp15VSvrhxu7XWecBcYD9QAJRrrb/Gjdt8glO1s8MyrjsGfHOPHnfruZ5KKT/gY+B+rXWFq+txJqXUZKBIa53u6lo6mQVIBV7RWg8HqnGPoYlTcow5XwkkAL0BX6XUja6tqkvosIzrjgGfC8Qe83UMxq91bkkpZcUI94Va608cmwuVUlGO16OAIlfV5wRjgSlKqWyM4bfzlVLv4t5tBuPfda7W+hfH1x9hBL47t/sCYK/Wulhr3Qh8AozBvdt8rFO1s8MyrjsG/Dqgv1IqQSnlgXExYqmLa3IKpZTCGJPdrrV+7piXlgKzHJ/PAj7t7NqcRWv9qNY6Rmsdj/F3+x+t9Y24cZsBtNYHgByl1EDHpknANty73fuBs5VSPo5/65MwrjO5c5uPdap2LgWuU0p5KqUSgP7A2nadQWvd7T6Ay4CdwG7gj66ux4ntHIfxq9kWYJPj4zIgFOOqe5bjzxBX1+qk9k8APnd87vZtBlKA9Y6/7yVAsLu3G3gS2AH8CrwDeLpjm4H3MK4zNGL00H/TUjuBPzryLRO4tL3nlaUKhBDCTXXHIRohhBCtIAEvhBBuSgJeCCHclAS8EEK4KQl4IYRwUxLwokdRStmUUpuO+eiwu0WVUvHHrhYohKtZXF2AEJ2sVmud4uoihOgM0oMXAlBKZSul/qaUWuv46OfY3kcptVIptcXxZ5xje6RSarFSarPjY4zjUGal1GuONc6/Vkp5u6xRoseTgBc9jfcJQzTXHvNahdZ6FPBPjBUtcXz+ttY6GVgIzHNsnwd8p7UehrFmTIZje3/gJa31WUAZcLWT2yPEKcmdrKJHUUpVaa39mtmeDZyvtd7jWODtgNY6VClVAkRprRsd2wu01mFKqWIgRmtdf8wx4oFvtPEAB5RSjwBWrfV/O79lQpxMevBCHKVP8fmp3tOc+mM+tyHXuYQLScALcdS1x/y5xvH5TxirWgLcAPzg+HwlcBcceX5sQGcVKURrSe9C9DTeSqlNx3z9ldb68FRJT6XULxgdn5mObXOA+UqphzCeuHSLY/t9wKtKqd9g9NTvwlgtUIguQ8bgheDIGHya1rrE1bUI0VFkiEYIIdyU9OCFEMJNSQ9eCCHclAS8EEK4KQl4IYRwUxLwQgjhpiTghRDCTf1/85sJDBAlXjsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_model(model, criterion, optimizer, step_lr_scheduler, num_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the network on the 10000 test images: 58.82 %\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        apple       0.85      0.72      0.78       100\n",
      "aquarium_fish       0.57      0.79      0.66       100\n",
      "         baby       0.47      0.43      0.45       100\n",
      "         bear       0.49      0.37      0.42       100\n",
      "       beaver       0.38      0.47      0.42       100\n",
      "          bed       0.66      0.61      0.63       100\n",
      "          bee       0.67      0.68      0.67       100\n",
      "       beetle       0.67      0.58      0.62       100\n",
      "      bicycle       0.72      0.71      0.72       100\n",
      "       bottle       0.66      0.68      0.67       100\n",
      "         bowl       0.52      0.38      0.44       100\n",
      "          boy       0.48      0.28      0.35       100\n",
      "       bridge       0.61      0.70      0.65       100\n",
      "          bus       0.56      0.49      0.52       100\n",
      "    butterfly       0.50      0.59      0.54       100\n",
      "        camel       0.56      0.59      0.58       100\n",
      "          can       0.67      0.62      0.65       100\n",
      "       castle       0.72      0.72      0.72       100\n",
      "  caterpillar       0.55      0.54      0.54       100\n",
      "       cattle       0.60      0.43      0.50       100\n",
      "        chair       0.85      0.79      0.82       100\n",
      "   chimpanzee       0.61      0.83      0.70       100\n",
      "        clock       0.62      0.55      0.58       100\n",
      "        cloud       0.67      0.77      0.72       100\n",
      "    cockroach       0.85      0.64      0.73       100\n",
      "        couch       0.44      0.47      0.45       100\n",
      "         crab       0.45      0.50      0.48       100\n",
      "    crocodile       0.47      0.39      0.43       100\n",
      "          cup       0.70      0.75      0.72       100\n",
      "     dinosaur       0.58      0.52      0.55       100\n",
      "      dolphin       0.55      0.49      0.52       100\n",
      "     elephant       0.60      0.52      0.56       100\n",
      "     flatfish       0.42      0.53      0.47       100\n",
      "       forest       0.59      0.55      0.57       100\n",
      "          fox       0.54      0.58      0.56       100\n",
      "         girl       0.39      0.39      0.39       100\n",
      "      hamster       0.72      0.50      0.59       100\n",
      "        house       0.56      0.63      0.59       100\n",
      "     kangaroo       0.49      0.49      0.49       100\n",
      "     keyboard       0.65      0.71      0.68       100\n",
      "         lamp       0.51      0.60      0.55       100\n",
      "   lawn_mower       0.74      0.74      0.74       100\n",
      "      leopard       0.52      0.65      0.58       100\n",
      "         lion       0.65      0.64      0.65       100\n",
      "       lizard       0.44      0.35      0.39       100\n",
      "      lobster       0.48      0.49      0.49       100\n",
      "          man       0.37      0.46      0.41       100\n",
      "   maple_tree       0.61      0.57      0.59       100\n",
      "   motorcycle       0.82      0.84      0.83       100\n",
      "     mountain       0.69      0.72      0.71       100\n",
      "        mouse       0.28      0.28      0.28       100\n",
      "     mushroom       0.55      0.59      0.57       100\n",
      "     oak_tree       0.61      0.68      0.64       100\n",
      "       orange       0.75      0.82      0.78       100\n",
      "       orchid       0.63      0.79      0.70       100\n",
      "        otter       0.28      0.30      0.29       100\n",
      "    palm_tree       0.78      0.79      0.79       100\n",
      "         pear       0.69      0.63      0.66       100\n",
      " pickup_truck       0.67      0.78      0.72       100\n",
      "    pine_tree       0.58      0.53      0.55       100\n",
      "        plain       0.80      0.79      0.79       100\n",
      "        plate       0.64      0.69      0.67       100\n",
      "        poppy       0.62      0.67      0.64       100\n",
      "    porcupine       0.58      0.49      0.53       100\n",
      "       possum       0.43      0.40      0.42       100\n",
      "       rabbit       0.43      0.33      0.37       100\n",
      "      raccoon       0.72      0.53      0.61       100\n",
      "          ray       0.40      0.57      0.47       100\n",
      "         road       0.79      0.94      0.86       100\n",
      "       rocket       0.71      0.64      0.67       100\n",
      "         rose       0.58      0.69      0.63       100\n",
      "          sea       0.78      0.69      0.73       100\n",
      "         seal       0.35      0.32      0.34       100\n",
      "        shark       0.42      0.52      0.47       100\n",
      "        shrew       0.33      0.34      0.33       100\n",
      "        skunk       0.78      0.82      0.80       100\n",
      "   skyscraper       0.67      0.85      0.75       100\n",
      "        snail       0.39      0.52      0.44       100\n",
      "        snake       0.51      0.37      0.43       100\n",
      "       spider       0.72      0.57      0.64       100\n",
      "     squirrel       0.43      0.35      0.38       100\n",
      "    streetcar       0.59      0.67      0.63       100\n",
      "    sunflower       0.88      0.84      0.86       100\n",
      " sweet_pepper       0.66      0.51      0.58       100\n",
      "        table       0.66      0.46      0.54       100\n",
      "         tank       0.77      0.67      0.72       100\n",
      "    telephone       0.74      0.64      0.68       100\n",
      "   television       0.67      0.68      0.67       100\n",
      "        tiger       0.65      0.56      0.60       100\n",
      "      tractor       0.63      0.74      0.68       100\n",
      "        train       0.69      0.66      0.67       100\n",
      "        trout       0.71      0.70      0.71       100\n",
      "        tulip       0.62      0.37      0.46       100\n",
      "       turtle       0.38      0.40      0.39       100\n",
      "     wardrobe       0.86      0.80      0.83       100\n",
      "        whale       0.59      0.60      0.60       100\n",
      "  willow_tree       0.57      0.54      0.56       100\n",
      "         wolf       0.50      0.59      0.54       100\n",
      "        woman       0.30      0.39      0.34       100\n",
      "         worm       0.52      0.68      0.59       100\n",
      "\n",
      "     accuracy                           0.59     10000\n",
      "    macro avg       0.59      0.59      0.59     10000\n",
      " weighted avg       0.59      0.59      0.59     10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "y_pred = []\n",
    "y_true = []\n",
    "\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        # auf cpu kopieren um in numpy array umwandeln zu könenn\n",
    "        y_pred.extend(predicted.cpu().numpy())\n",
    "        y_true.extend(labels.cpu().numpy())\n",
    "        correct += (predicted == labels).sum().item()\n",
    "        del images, labels, outputs\n",
    "\n",
    "    print('Accuracy of the network on the {} test images: {} %'.format(10000, 100 * correct / total))  \n",
    "\n",
    "target_names = fine_label_names\n",
    "classification_report_result = classification_report(y_true, y_pred, target_names=target_names)\n",
    "print(classification_report_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4\" ><caption>ResNet Top 5 labels precision</caption><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >precision</th>        <th class=\"col_heading level0 col1\" >recall</th>        <th class=\"col_heading level0 col2\" >f1-score</th>        <th class=\"col_heading level0 col3\" >support</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4level0_row0\" class=\"row_heading level0 row0\" >sunflower</th>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row0_col0\" class=\"data row0 col0\" >0.875000</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row0_col1\" class=\"data row0 col1\" >0.840000</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row0_col2\" class=\"data row0 col2\" >0.857143</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row0_col3\" class=\"data row0 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4level0_row1\" class=\"row_heading level0 row1\" >wardrobe</th>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row1_col0\" class=\"data row1 col0\" >0.860215</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row1_col1\" class=\"data row1 col1\" >0.800000</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row1_col2\" class=\"data row1 col2\" >0.829016</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row1_col3\" class=\"data row1 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4level0_row2\" class=\"row_heading level0 row2\" >cockroach</th>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row2_col0\" class=\"data row2 col0\" >0.853333</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row2_col1\" class=\"data row2 col1\" >0.640000</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row2_col2\" class=\"data row2 col2\" >0.731429</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row2_col3\" class=\"data row2 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4level0_row3\" class=\"row_heading level0 row3\" >chair</th>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row3_col0\" class=\"data row3 col0\" >0.849462</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row3_col1\" class=\"data row3 col1\" >0.790000</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row3_col2\" class=\"data row3 col2\" >0.818653</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row3_col3\" class=\"data row3 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4level0_row4\" class=\"row_heading level0 row4\" >apple</th>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row4_col0\" class=\"data row4 col0\" >0.847059</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row4_col1\" class=\"data row4 col1\" >0.720000</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row4_col2\" class=\"data row4 col2\" >0.778378</td>\n",
       "                        <td id=\"T_23137c52_0574_11ee_8b41_3e9c7d8f0ce4row4_col3\" class=\"data row4 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8c6ed9d710>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "classification_report_result_dict = classification_report(y_true, y_pred, target_names=target_names, output_dict=True)\n",
    "df_classification_report = pd.DataFrame(classification_report_result_dict).transpose()\n",
    "df_classification_report[0:len(target_names)]\n",
    "\n",
    "# Find best classified labels\n",
    "sorted_df_precision = df_classification_report.sort_values('precision', ascending=False)\n",
    "top5_sorted_precision = sorted_df_precision[0:5]\n",
    "header_precision = \"ResNet Top 5 labels precision\"\n",
    "sorted_df_precision = top5_sorted_precision.style.set_caption(header_precision)\n",
    "sorted_df_precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4\" ><caption>ResNet worst 5 labels precision</caption><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >precision</th>        <th class=\"col_heading level0 col1\" >recall</th>        <th class=\"col_heading level0 col2\" >f1-score</th>        <th class=\"col_heading level0 col3\" >support</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4level0_row0\" class=\"row_heading level0 row0\" >mouse</th>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row0_col0\" class=\"data row0 col0\" >0.282828</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row0_col1\" class=\"data row0 col1\" >0.280000</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row0_col2\" class=\"data row0 col2\" >0.281407</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row0_col3\" class=\"data row0 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4level0_row1\" class=\"row_heading level0 row1\" >otter</th>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row1_col0\" class=\"data row1 col0\" >0.283019</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row1_col1\" class=\"data row1 col1\" >0.300000</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row1_col2\" class=\"data row1 col2\" >0.291262</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row1_col3\" class=\"data row1 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4level0_row2\" class=\"row_heading level0 row2\" >woman</th>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row2_col0\" class=\"data row2 col0\" >0.300000</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row2_col1\" class=\"data row2 col1\" >0.390000</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row2_col2\" class=\"data row2 col2\" >0.339130</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row2_col3\" class=\"data row2 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4level0_row3\" class=\"row_heading level0 row3\" >shrew</th>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row3_col0\" class=\"data row3 col0\" >0.330097</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row3_col1\" class=\"data row3 col1\" >0.340000</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row3_col2\" class=\"data row3 col2\" >0.334975</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row3_col3\" class=\"data row3 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4level0_row4\" class=\"row_heading level0 row4\" >seal</th>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row4_col0\" class=\"data row4 col0\" >0.351648</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row4_col1\" class=\"data row4 col1\" >0.320000</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row4_col2\" class=\"data row4 col2\" >0.335079</td>\n",
       "                        <td id=\"T_237c728e_0574_11ee_8b41_3e9c7d8f0ce4row4_col3\" class=\"data row4 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8c6ecf5bd0>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_df_precision = df_classification_report.sort_values('precision')\n",
    "worst5_sorted_precision = sorted_df_precision[0:5]\n",
    "header_precision = \"ResNet worst 5 labels precision\"\n",
    "sorted_df_precision = worst5_sorted_precision.style.set_caption(header_precision)\n",
    "sorted_df_precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4\" ><caption>ResNet worst 5 labels f1-score</caption><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >precision</th>        <th class=\"col_heading level0 col1\" >recall</th>        <th class=\"col_heading level0 col2\" >f1-score</th>        <th class=\"col_heading level0 col3\" >support</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4level0_row0\" class=\"row_heading level0 row0\" >mouse</th>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row0_col0\" class=\"data row0 col0\" >0.282828</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row0_col1\" class=\"data row0 col1\" >0.280000</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row0_col2\" class=\"data row0 col2\" >0.281407</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row0_col3\" class=\"data row0 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4level0_row1\" class=\"row_heading level0 row1\" >otter</th>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row1_col0\" class=\"data row1 col0\" >0.283019</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row1_col1\" class=\"data row1 col1\" >0.300000</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row1_col2\" class=\"data row1 col2\" >0.291262</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row1_col3\" class=\"data row1 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4level0_row2\" class=\"row_heading level0 row2\" >shrew</th>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row2_col0\" class=\"data row2 col0\" >0.330097</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row2_col1\" class=\"data row2 col1\" >0.340000</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row2_col2\" class=\"data row2 col2\" >0.334975</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row2_col3\" class=\"data row2 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4level0_row3\" class=\"row_heading level0 row3\" >seal</th>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row3_col0\" class=\"data row3 col0\" >0.351648</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row3_col1\" class=\"data row3 col1\" >0.320000</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row3_col2\" class=\"data row3 col2\" >0.335079</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row3_col3\" class=\"data row3 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4level0_row4\" class=\"row_heading level0 row4\" >woman</th>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row4_col0\" class=\"data row4 col0\" >0.300000</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row4_col1\" class=\"data row4 col1\" >0.390000</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row4_col2\" class=\"data row4 col2\" >0.339130</td>\n",
       "                        <td id=\"T_23b24e9a_0574_11ee_8b41_3e9c7d8f0ce4row4_col3\" class=\"data row4 col3\" >100.000000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8c6ed08dd0>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_df_f1_score = df_classification_report.sort_values('f1-score')\n",
    "worst5_sorted_f1_score = sorted_df_f1_score[0:5]\n",
    "header_f1_score = \"ResNet worst 5 labels f1-score\"\n",
    "sorted_df_f1_score = worst5_sorted_f1_score.style.set_caption(header_f1_score)\n",
    "sorted_df_f1_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary:\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### - ResNet architecture made a improve \n",
    "#### - Less parameter and higher accurancy on test data\n",
    "#### - Similar classes are classified with a low precision and f1-score: mouse, otter \n",
    "#### - ResNet structure made a improve on CIFAR100 compared to VGG architecture"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}