Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "from cells import *\n",
    "import torchvision.transforms as transforms\n",
    "from torchvision.datasets import CIFAR10\n",
    "from torch.utils.data import DataLoader\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.tensorboard import SummaryWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CIFAR dataset preprocessing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# define epochs, batch_size to use and learning rate\n",
    "num_epochs = 8\n",
    "batch_size = 128\n",
    "learning_rate = 0.025\n",
    "\n",
    "#define transforms to use while making data\n",
    "img_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "])\n",
    "\n",
    "train_dataset = CIFAR10('./data', transform=img_transform, download = True, train = True)\n",
    "\n",
    "test_dataset = CIFAR10('./data', transform=img_transform, download = True, train = False)\n",
    "\n",
    "# create dataloaders for train and test which will be used later to get batches during training \n",
    "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dataset sizes for both train and test\n",
    "dataset_sizes = {'train': len(train_dataset), 'test': len(test_dataset)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Network\n",
    "Construct a network using reduction and normal cells. The architecture is same as shown in the papers \n",
    "https://arxiv.org/pdf/1802.01548.pdf  and https://arxiv.org/pdf/1712.00559.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network(nn.Module):\n",
    "    \n",
    "    def __init__(self, total_layers = 8, initial_ch = 32):\n",
    "        super(Network, self).__init__()\n",
    "        \n",
    "        assert total_layers < 10\n",
    "        \n",
    "        # the first layer is not either type of a cell, it's a normal convolution layer to create initial feature map\n",
    "        # which will be used by normal and reduction cells further\n",
    "        self.first_layer = nn.Sequential(nn.Conv2d(3, initial_ch, 3, stride = 1, padding = 1),\n",
    "                                   nn.ReLU())\n",
    "        \n",
    "        c_kminus1, c_kminus2 = initial_ch, initial_ch\n",
    "        \n",
    "        reduction = False\n",
    "        #cells will be added to the ModuleDict by torch\n",
    "        self.cells = nn.ModuleDict()\n",
    "        for i in range(1, total_layers+ 1):\n",
    "            if i % 3 == 0:\n",
    "                # add reduction cell at every third layer in the network \n",
    "                self.cells['cell_reduction' + str(i)] = Reduction_cell(c_kminus1, c_kminus2 * 2)\n",
    "                reduction = True \n",
    "                c_kminus1 = c_kminus1 * 2\n",
    "            else:\n",
    "                #add normal cell to network\n",
    "                self.cells['cell_normal' + str(i)] = Normal_cell(c_kminus1, c_kminus2, reduction)\n",
    "                reduction = False\n",
    "                if c_kminus1 != c_kminus2:\n",
    "                    c_kminus2 *= 2\n",
    "                c_kminus1  = c_kminus1\n",
    "                c_kminus2 = c_kminus2\n",
    "                \n",
    "        # after all the cells, global average pooling reduces the feature map size to 4x4\n",
    "        self.global_pooling = nn.AdaptiveAvgPool2d(4)\n",
    "        # two fully connected layers are added to make prediction in one of 10 classes\n",
    "        self.final = nn.Sequential(nn.Linear(c_kminus1*4*4, 2048),\n",
    "                                   nn.ReLU(),\n",
    "                                   nn.Linear(2048, 10))\n",
    "        \n",
    "        \n",
    "    def forward(self, x):\n",
    "        k_minus_1 = self.first_layer(x)\n",
    "        k_minus_2 = self.first_layer(x)\n",
    "        \n",
    "        for item, val in (self.cells).items():\n",
    "            \n",
    "            out = val(k_minus_1, k_minus_2)\n",
    "            \n",
    "            k_minus_2 = k_minus_1\n",
    "            k_minus_1 = out\n",
    "        \n",
    "        out = self.global_pooling(out)\n",
    "        out = self.final(out.view(out.size(0), -1))\n",
    "        return F.softmax(out,1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/harsh/Desktop/sharad/cells.py:98: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  assert k_minus_1.size(1) == k_minus_2.size(1)\n",
      "/home/harsh/Desktop/sharad/cells.py:125: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  assert k_minus_1.size(1) == k_minus_2.size(1)\n",
      "/home/harsh/Desktop/sharad/cells.py:126: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  assert k_minus_1.size(2) == k_minus_2.size(2)\n"
     ]
    }
   ],
   "source": [
    "#initialize network a\n",
    "model = Network().cuda()\n",
    "model.train()\n",
    "\n",
    "images, labels = next(iter(train_dataloader))\n",
    "images = images.to(device)\n",
    "grid = torchvision.utils.make_grid(images)\n",
    "\n",
    "# using tensorboard to visualize a batch of images and Network\n",
    "tb = SummaryWriter()\n",
    "tb.add_image('images', grid)\n",
    "tb.add_graph(model, images)\n",
    "tb.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training \n",
    "For training, to get the good results, paramters used in paper itself are used like learning rate, optimizer, scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define loss, optimizer and scheduler\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), learning_rate, momentum = 0.9, weight_decay= 3e-4)\n",
    "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(num_epochs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 1 ---------------\n",
      "0 391 Loss: 2.304 | Acc: 8.594% (11/128)\n",
      "100 391 Loss: 2.196 | Acc: 25.511% (3298/12928)\n",
      "200 391 Loss: 2.119 | Acc: 33.555% (8633/25728)\n",
      "300 391 Loss: 2.074 | Acc: 38.180% (14710/38528)\n",
      "--------------- Epoch 2 ---------------\n",
      "0 391 Loss: 1.970 | Acc: 50.000% (64/128)\n",
      "100 391 Loss: 1.906 | Acc: 55.608% (7189/12928)\n",
      "200 391 Loss: 1.895 | Acc: 56.689% (14585/25728)\n",
      "300 391 Loss: 1.884 | Acc: 57.714% (22236/38528)\n",
      "--------------- Epoch 3 ---------------\n",
      "0 391 Loss: 1.780 | Acc: 69.531% (89/128)\n",
      "100 391 Loss: 1.813 | Acc: 64.944% (8396/12928)\n",
      "200 391 Loss: 1.807 | Acc: 65.396% (16825/25728)\n",
      "300 391 Loss: 1.800 | Acc: 66.147% (25485/38528)\n",
      "--------------- Epoch 4 ---------------\n",
      "0 391 Loss: 1.769 | Acc: 70.312% (90/128)\n",
      "100 391 Loss: 1.735 | Acc: 73.004% (9438/12928)\n",
      "200 391 Loss: 1.735 | Acc: 72.952% (18769/25728)\n",
      "300 391 Loss: 1.734 | Acc: 73.009% (28129/38528)\n",
      "--------------- Epoch 5 ---------------\n",
      "0 391 Loss: 1.745 | Acc: 72.656% (93/128)\n",
      "100 391 Loss: 1.696 | Acc: 76.926% (9945/12928)\n",
      "200 391 Loss: 1.691 | Acc: 77.383% (19909/25728)\n",
      "300 391 Loss: 1.691 | Acc: 77.372% (29810/38528)\n",
      "--------------- Epoch 6 ---------------\n",
      "0 391 Loss: 1.655 | Acc: 82.031% (105/128)\n",
      "100 391 Loss: 1.649 | Acc: 81.791% (10574/12928)\n",
      "200 391 Loss: 1.652 | Acc: 81.312% (20920/25728)\n",
      "300 391 Loss: 1.649 | Acc: 81.621% (31447/38528)\n",
      "--------------- Epoch 7 ---------------\n",
      "0 391 Loss: 1.715 | Acc: 75.781% (97/128)\n",
      "100 391 Loss: 1.615 | Acc: 85.071% (10998/12928)\n",
      "200 391 Loss: 1.614 | Acc: 85.234% (21929/25728)\n",
      "300 391 Loss: 1.612 | Acc: 85.405% (32905/38528)\n",
      "--------------- Epoch 8 ---------------\n",
      "0 391 Loss: 1.622 | Acc: 83.594% (107/128)\n",
      "100 391 Loss: 1.590 | Acc: 87.778% (11348/12928)\n",
      "200 391 Loss: 1.588 | Acc: 87.990% (22638/25728)\n",
      "300 391 Loss: 1.588 | Acc: 87.936% (33880/38528)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "train_loss = 0\n",
    "correct = 0\n",
    "total = 0\n",
    "for epoch in range(num_epochs):\n",
    "    \n",
    "    print('-'*15 , 'Epoch' ,epoch +1,'-'*15)\n",
    "    correct = 0\n",
    "    train_loss = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(train_dataloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        train_loss += loss.item()\n",
    "        _, predicted = outputs.max(1)\n",
    "        total += targets.size(0)\n",
    "        correct += predicted.eq(targets).sum().item()\n",
    "\n",
    "        if batch_idx % 100 == 0:\n",
    "            print(batch_idx, len(train_dataloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'\n",
    "                % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))\n",
    "            \n",
    "    scheduler.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Network(\n",
       "  (first_layer): Sequential(\n",
       "    (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): ReLU()\n",
       "  )\n",
       "  (cells): ModuleDict(\n",
       "    (cell_normal1): Normal_cell(\n",
       "      (maintain_ch): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (6): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (6): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to2): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (6): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (6): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (6): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (zeroto3): dilation_separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (cell_normal2): Normal_cell(\n",
       "      (maintain_ch): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (6): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (6): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to2): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (6): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (6): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "          (6): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (zeroto3): dilation_separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), groups=32)\n",
       "          (2): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (cell_reduction3): Reduction_cell(\n",
       "      (double_ch): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (pool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "      (maintain_ch): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (cell_normal4): Normal_cell(\n",
       "      (reduce_spatial): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(32, 64, kernel_size=(1, 1), stride=(2, 2))\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (maintain_ch): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (6): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (6): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to2): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (6): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (6): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (6): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (zeroto3): dilation_separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (cell_normal5): Normal_cell(\n",
       "      (maintain_ch): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (6): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (6): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to2): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (6): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (6): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "          (6): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (zeroto3): dilation_separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), groups=64)\n",
       "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (cell_reduction6): Reduction_cell(\n",
       "      (double_ch): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (pool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "      (maintain_ch): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (cell_normal7): Normal_cell(\n",
       "      (reduce_spatial): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2))\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (maintain_ch): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (6): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (6): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to2): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (6): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (6): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (6): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (zeroto3): dilation_separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (cell_normal8): Normal_cell(\n",
       "      (maintain_ch): Compatible(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (6): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (6): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus1to2): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (6): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to0): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (6): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (kminus2to1): Separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (4): ReLU()\n",
       "          (5): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "          (6): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (zeroto3): dilation_separable(\n",
       "        (layer): Sequential(\n",
       "          (0): ReLU()\n",
       "          (1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), groups=128)\n",
       "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (global_pooling): AdaptiveAvgPool2d(output_size=4)\n",
       "  (final): Sequential(\n",
       "    (0): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Linear(in_features=2048, out_features=10, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss  1.6982161356852605\n",
      "Accuracy  78.62\n"
     ]
    }
   ],
   "source": [
    "\n",
    "test_loss = 0\n",
    "correct = 0\n",
    "total = 0\n",
    "for batch_idx, (inputs, targets) in enumerate(test_dataloader):\n",
    "    \n",
    "    inputs, targets = inputs.to(device), targets.to(device)\n",
    "    outputs = model(inputs)\n",
    "    loss = criterion(outputs, targets)\n",
    "\n",
    "    test_loss += loss.item()\n",
    "    _, predicted = outputs.max(1)\n",
    "    total += targets.size(0)\n",
    "    correct += predicted.eq(targets).sum().item()\n",
    "\n",
    "print('loss ', test_loss/batch_idx)\n",
    "print('Accuracy ', 100. * correct/dataset_sizes['test'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}