Week11_Pytorch_BatchNorm.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0eelkFb3A9_C"
      },
      "source": [
        "# Batch Normalization in Pytorch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "HZ0W7icSA9_H"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pSSsGJAeA9_G"
      },
      "source": [
        "## Prepare dataset\n",
        "- MNIST dataset\n",
        "  - 28x28 image"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "D-pO_7U8A9_J"
      },
      "outputs": [],
      "source": [
        "from torchvision import datasets\n",
        "from torchvision.transforms import ToTensor\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "# get the MNIST datasets\n",
        "# transform: convert data to FloatTensor\n",
        "train_data = datasets.MNIST(root='data', train=True, download=True,\n",
        "                            transform=ToTensor())\n",
        "test_data = datasets.MNIST(root='data', train=False, download=True,\n",
        "                           transform=ToTensor())\n",
        "\n",
        "# number of samples per batch\n",
        "batch_size = 64\n",
        "\n",
        "# DataLoader\n",
        "# num_workers: number of subprocesses to use for data loading\n",
        "train_loader = DataLoader(train_data, batch_size=batch_size, num_workers=0)\n",
        "test_loader = DataLoader(test_data, batch_size=batch_size, num_workers=0)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(train_loader))\n",
        "print(len(test_loader))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KUPyxQApnYsM",
        "outputId": "83ac35f9-a2b2-4ff2-8126-0e0e1f475033"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "938\n",
            "157\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rtolug7ZA9_K"
      },
      "source": [
        "### Visualize the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 329
        },
        "id": "gX5LLMxbA9_L",
        "outputId": "8e7b11bd-7007-4ddd-ce70-c1c6ebb37d9d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(28, 28)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7ca2a0c3df90>"
            ]
          },
          "metadata": {},
          "execution_count": 16
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 300x300 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAEUCAYAAADuhRlEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVSUlEQVR4nO3df3AUd/3H8VcC5PjR5GIK5IiQNFQFLQIOlpiCFGxKGkcGKM5Y1Bk6MmDrpSOg1IljQaptWlB0sNjyhybFEVKZKSD8kYqBJONI4iQFEZEMICPB5FKL5i6kJNDc5/uH9r69EnZz5BPuDp6Pmc9Mb9+f231323vNZnfvNsUYYwQAFqXGuwEAtx+CBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwLrh8W7gg8LhsNra2pSenq6UlJR4twPgf4wx6urqUk5OjlJTXY5JzBB56aWXTF5envF4PGb27NmmsbFxQO9rbW01khgMRoKO1tZW18/xkARLVVWVSUtLM7/85S/NX//6V7Nq1SqTmZlpOjo6XN/b2dkZ9x3HYDBuPDo7O10/x0MSLLNnzzZ+vz/yuq+vz+Tk5Jjy8nLX9waDwbjvOAaDceMRDAZdP8fWT95evXpVzc3NKioqiixLTU1VUVGRjh49et383t5ehUKhqAEguVkPlrffflt9fX3Kzs6OWp6dna1AIHDd/PLycnm93siYNGmS7ZYA3GJxv9xcVlamYDAYGa2trfFuCcAgWb/cPHbsWA0bNkwdHR1Ryzs6OuTz+a6b7/F45PF4bLcBII6sH7GkpaVp1qxZqqmpiSwLh8OqqalRYWGh7c0BSESDuvxzA1VVVcbj8ZjKykpz6tQps3r1apOZmWkCgYDre7kqxGAk9hjIVaEhufP2S1/6kv71r39pw4YNCgQCmjlzpqqrq687oQvg9pRiTGL9mHYoFJLX6413GwBuIBgMKiMjw3FO3K8KAbj9ECwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsC64bZX+P3vf1+bNm2KWjZlyhSdPn3a9qZwCwwbNsyx7vV6h7yH0tJSx/ro0aMd61OmTHHdht/vd6z/6Ec/cqwvX77csd7T0+PawwsvvOBY/+DnKpFZDxZJuu+++/T73//+/zcyfEg2AyBBDcknfvjw4fL5fEOxagBJYEjOsZw5c0Y5OTmaPHmyvvKVr+jChQtDsRkACcr6EUtBQYEqKys1ZcoUtbe3a9OmTfrsZz+rkydPKj09/br5vb296u3tjbwOhUK2WwJwi1kPlpKSksg/T58+XQUFBcrLy9NvfvMbrVy58rr55eXlSXVSCoC7Ib/cnJmZqY997GM6e/Zsv/WysjIFg8HIaG1tHeqWAAyxIQ+Wy5cv69y5c5owYUK/dY/Ho4yMjKgBILlZ/1Po29/+thYtWqS8vDy1tbVp48aNGjZsmOt1flwvNzfXsZ6WluZYf+CBB1y3MXfuXMd6ZmamY33ZsmWu24i3ixcvus7Ztm2bY33p0qWO9a6uLsf6n//8Z9ce6urqXOckC+vBcvHiRS1fvlyXLl3SuHHjNHfuXDU0NGjcuHG2NwUgQVkPlqqqKturBJBk+K4QAOsIFgDWESwArCNYAFhHsACwjmABYF2KMcbEu4n3C4VCt+THgxLBzJkzHeuHDx92rN8p+8lNOBx2rH/ta19zXcfly5cH1UN7e7tj/T//+Y/rOlpaWgbVw60SDAZd75DniAWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB0P/Ikjt6cXXLp0ybGeDPexNDY2us7p7Ox0rC9YsMCxfvXqVcf6r371K9ceYBdHLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA67iPJY7+/e9/O9bXr1/vWP/CF77gWD927JhrD24P6nJz/Phxx/rDDz/suo7u7m7H+n333edY/+Y3v+m6DdxaHLEAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKzjuUJJzO3ZLl1dXa7r2LFjh2N95cqVjvWvfvWrjvXdu3e79oDkMiTPFaqvr9eiRYuUk5OjlJQU7du3L6pujNGGDRs0YcIEjRo1SkVFRTpz5kysmwGQxGIOlu7ubs2YMUPbt2/vt75582Zt27ZNr7zyihobGzVmzBgVFxerp6dn0M0CSA4x39JfUlKikpKSfmvGGP30pz/V9773PS1evFiStHPnTmVnZ2vfvn167LHHBtctgKRg9eTt+fPnFQgEVFRUFFnm9XpVUFCgo0eP9vue3t5ehUKhqAEguVkNlkAgIEnKzs6OWp6dnR2pfVB5ebm8Xm9kTJo0yWZLAOIg7peby8rKFAwGI6O1tTXeLQEYJKvB4vP5JEkdHR1Ryzs6OiK1D/J4PMrIyIgaAJKb1WDJz8+Xz+dTTU1NZFkoFFJjY6MKCwttbgpAAov5qtDly5d19uzZyOvz58/r+PHjysrKUm5urtasWaMf/vCH+uhHP6r8/Hw988wzysnJ0ZIlS2z2DcnKie5gMDio969atcqx/tprr7muIxwOD6oHJJ6Yg6WpqSnqyXTr1q2TJK1YsUKVlZV6+umn1d3drdWrV6uzs1Nz585VdXW1Ro4caa9rAAkt5mCZP3++nL4FkJKSomeffVbPPvvsoBoDkLziflUIwO2HYAFgHcECwDqCBYB1BAsA6/ihpzvcmDFjHOsHDhxwrD/44IOO9Rt9E/79fve737nOQeIYkh96AgA3BAsA6wgWANYRLACsI1gAWEewALCOYAFgHfexwNG9997rWH/zzTcd652dna7bOHLkiGO9qanJsX6jR9G8J8H+F0963McCIC4IFgDWESwArCNYAFhHsACwjmABYB3BAsA67mPBoCxdutSxXlFR4bqO9PT0QfXw3e9+17G+c+dO13W0t7cPqoc7CfexAIgLggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYF3MN8jV19dry5Ytam5uVnt7u/bu3aslS5ZE6o8//rheffXVqPcUFxerurp6QOvnBrnby7Rp01znbN261bH+0EMPDaqHHTt2uM557rnnHOv//Oc/B9XD7WRIbpDr7u7WjBkzHH+165FHHlF7e3tk7N69O9bNAEhiw2N9Q0lJietjMz0ej3w+3003BSC5Dck5ltraWo0fP15TpkzRk08+qUuXLt1wbm9vr0KhUNQAkNysB8sjjzyinTt3qqamRi+++KLq6upUUlKivr6+fueXl5fL6/VGxqRJk2y3BOAWi/lPITePPfZY5J8/+clPavr06br33ntVW1vb70m4srIyrVu3LvI6FAoRLkCSG/LLzZMnT9bYsWN19uzZfusej0cZGRlRA0ByG/JguXjxoi5duqQJEyYM9aYAJIiY72O5fPly5OjjU5/6lLZu3aoFCxYoKytLWVlZ2rRpk5YtWyafz6dz587p6aefVldXl/7yl7/I4/G4rp/7WO48mZmZjvVFixY51t1+TColJcW1h8OHDzvWH374Ydd13CkGch9LzOdYmpqatGDBgsjr986PrFixQi+//LJOnDihV199VZ2dncrJydHChQv1gx/8YEChAuD2EHOwzJ8/3/GRlW+88cagGgKQ/PiuEADrCBYA1hEsAKwjWABYR7AAsI4HliHp9fb2OtaHD3e/+Pnuu+861ouLix3rtbW1rtu4XfDAMgBxQbAAsI5gAWAdwQLAOoIFgHUECwDrCBYA1ln/aUrg/aZPn+4654tf/KJj/f7773esD+Q+FTenTp1yrNfX1w96G3cSjlgAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACs4wY5OJoyZYpjvbS01LH+6KOPum7D5/PF1FOs+vr6XOe0t7c71sPhsK127ggcsQCwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArIvpPpby8nK9/vrrOn36tEaNGqUHHnhAL774YtS9Dj09PfrWt76lqqoq9fb2qri4WD//+c+VnZ1tvXk4G8j9IcuXL3esu92ncs8998TS0pBoampyrD/33HOu6/jtb39rqx0oxiOWuro6+f1+NTQ06NChQ7p27ZoWLlyo7u7uyJy1a9fqwIED2rNnj+rq6tTW1jagm6QA3D5iOmKprq6Oel1ZWanx48erublZ8+bNUzAY1C9+8Qvt2rVLn/vc5yRJFRUV+vjHP66GhgZ95jOfsdc5gIQ1qHMswWBQkpSVlSVJam5u1rVr11RUVBSZM3XqVOXm5uro0aP9rqO3t1ehUChqAEhuNx0s4XBYa9as0Zw5czRt2jRJUiAQUFpamjIzM6PmZmdnKxAI9Lue8vJyeb3eyJg0adLNtgQgQdx0sPj9fp08eVJVVVWDaqCsrEzBYDAyWltbB7U+APF3U99uLi0t1cGDB1VfX6+JEydGlvt8Pl29elWdnZ1RRy0dHR03vELh8Xjk8Xhupg0ACSqmIxZjjEpLS7V3714dPnxY+fn5UfVZs2ZpxIgRqqmpiSxraWnRhQsXVFhYaKdjAAkvpiMWv9+vXbt2af/+/UpPT4+cN/F6vRo1apS8Xq9WrlypdevWKSsrSxkZGXrqqadUWFjIFaGb4Hbvzyc+8QnH+ksvveS6jalTp8bU01BobGx0rG/ZssWxvn//fsc6v6Vy68UULC+//LIkaf78+VHLKyoq9Pjjj0uSfvKTnyg1NVXLli2LukEOwJ0jpmAxxrjOGTlypLZv367t27ffdFMAkhvfFQJgHcECwDqCBYB1BAsA6wgWANbxXKEh8t4XM53s2LHDsT5z5kzH+uTJk2NpaUj88Y9/dKz/+Mc/dl3HG2+84Vi/cuVKTD0h/jhiAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI4b5G6goKDAsb5+/XrH+uzZs1238eEPfzimnobCO++841jftm2bY/355593rL//mVO4c3DEAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjvtYbmDp0qWDqttw6tQpx/rBgwcd6++++67rNtx+iKmzs9N1HcAHccQCwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALDPxOD55583n/70p81dd91lxo0bZxYvXmxOnz4dNefBBx80kqLG17/+9QFvIxgMXvd+BoOROCMYDLp+jmM6Yqmrq5Pf71dDQ4MOHTqka9euaeHChdf9mM+qVavU3t4eGZs3b45lMwCSXEx33lZXV0e9rqys1Pjx49Xc3Kx58+ZFlo8ePVo+n89OhwCSzqDOsQSDQUnXP6f417/+tcaOHatp06aprKzM9ecPAdxebvq7QuFwWGvWrNGcOXM0bdq0yPIvf/nLysvLU05Ojk6cOKHvfOc7amlp0euvv97venp7e9Xb2xt5HQqFbrYlAIkilpO37/fEE0+YvLw809ra6jivpqbGSDJnz57tt75x48a4n4xiMBgDHwM5eXtTweL3+83EiRPN3//+d9e5ly9fNpJMdXV1v/Wenh4TDAYjo7W1Ne47jsFg3HgMJFhi+lPIGKOnnnpKe/fuVW1trfLz813fc/z4cUnShAkT+q17PB55PJ5Y2gCQ4GIKFr/fr127dmn//v1KT09XIBCQJHm9Xo0aNUrnzp3Trl279PnPf1533323Tpw4obVr12revHmaPn36kPwLAEhAsfwJpBscGlVUVBhjjLlw4YKZN2+eycrKMh6Px3zkIx8x69evH9Ch03u4QY7BSOwxkM9zyv8CI2GEQiF5vd54twHgBoLBoDIyMhzn8F0hANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgXcIFS4J92RrABwzkM5pwwdLV1RXvFgA4GMhnNOF+jyUcDqutrU3p6elKSUlRKBTSpEmT1Nra6vobEHDGvrTjTt2Pxhh1dXUpJydHqanOxyQ3/fiPoZKamqqJEydetzwjI+OO+o84lNiXdtyJ+3GgP8KWcH8KAUh+BAsA6xI+WDwejzZu3MgjQixgX9rBfnSXcCdvASS/hD9iAZB8CBYA1hEsAKwjWABYl/DBsn37dt1zzz0aOXKkCgoK9Kc//SneLSW8+vp6LVq0SDk5OUpJSdG+ffui6sYYbdiwQRMmTNCoUaNUVFSkM2fOxKfZBFZeXq77779f6enpGj9+vJYsWaKWlpaoOT09PfL7/br77rt11113admyZero6IhTx4kjoYPltdde07p167Rx40a9+eabmjFjhoqLi/XWW2/Fu7WE1t3drRkzZmj79u391jdv3qxt27bplVdeUWNjo8aMGaPi4mL19PTc4k4TW11dnfx+vxoaGnTo0CFdu3ZNCxcuVHd3d2TO2rVrdeDAAe3Zs0d1dXVqa2vTo48+GseuE0QsD4W/1WbPnm38fn/kdV9fn8nJyTHl5eVx7Cq5SDJ79+6NvA6Hw8bn85ktW7ZElnV2dhqPx2N2794dhw6Tx1tvvWUkmbq6OmPMf/fbiBEjzJ49eyJz/va3vxlJ5ujRo/FqMyEk7BHL1atX1dzcrKKiosiy1NRUFRUV6ejRo3HsLLmdP39egUAgar96vV4VFBSwX10Eg0FJUlZWliSpublZ165di9qXU6dOVW5u7h2/LxM2WN5++2319fUpOzs7anl2drYCgUCcukp+7+079mtswuGw1qxZozlz5mjatGmS/rsv09LSlJmZGTWXfZmA324GEpHf79fJkyf1hz/8Id6tJIWEPWIZO3ashg0bdt0Z9o6ODvl8vjh1lfze23fs14ErLS3VwYMHdeTIkaif9PD5fLp69ao6Ozuj5rMvEzhY0tLSNGvWLNXU1ESWhcNh1dTUqLCwMI6dJbf8/Hz5fL6o/RoKhdTY2Mh+/QBjjEpLS7V3714dPnxY+fn5UfVZs2ZpxIgRUfuypaVFFy5cYF/G++yxk6qqKuPxeExlZaU5deqUWb16tcnMzDSBQCDerSW0rq4uc+zYMXPs2DEjyWzdutUcO3bM/OMf/zDGGPPCCy+YzMxMs3//fnPixAmzePFik5+fb65cuRLnzhPLk08+abxer6mtrTXt7e2R8c4770TmPPHEEyY3N9ccPnzYNDU1mcLCQlNYWBjHrhNDQgeLMcb87Gc/M7m5uSYtLc3Mnj3bNDQ0xLulhHfkyBEj6bqxYsUKY8x/Lzk/88wzJjs723g8HvPQQw+ZlpaW+DadgPrbh5JMRUVFZM6VK1fMN77xDfOhD33IjB492ixdutS0t7fHr+kEwc8mALAuYc+xAEheBAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACs+z/cCStkNimn9wAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "# one batch of training images\n",
        "images, labels = next(iter(train_loader))\n",
        "# to numpy array\n",
        "images = images.numpy()\n",
        "\n",
        "# one image from the batch\n",
        "img = np.squeeze(images[0])\n",
        "# image shape\n",
        "print(img.shape)\n",
        "\n",
        "fig = plt.figure(figsize = (3,3))\n",
        "ax = fig.add_subplot(111)\n",
        "ax.imshow(img, cmap='gray')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PBbYBg5_A9_M"
      },
      "source": [
        "## Model class\n",
        "- a simple MLP for classification\n",
        "\n",
        "- add batch normalization to layers\n",
        "  - layers with batch normalization do **not** include a bias term\n",
        "  - use `nn.BatchNorm1d()` to operate on linear layer outputs\n",
        "  - add the batch normalization layer **before** calling the activation function\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "iOvSo8CFA9_N"
      },
      "outputs": [],
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "class NeuralNet(nn.Module):\n",
        "    def __init__(self, use_batch_norm, input_size=784, hidden_dim=256, output_size=10):\n",
        "        # init super(=nn.Module)\n",
        "        super(NeuralNet, self).__init__()\n",
        "\n",
        "        # input_size: number of features (28x28 image)\n",
        "        self.input_size = input_size\n",
        "        self.hidden_dim = hidden_dim\n",
        "        # output_size: number of classes (0~9)\n",
        "        self.output_size = output_size\n",
        "        # use_batch_norm: whether or not this network uses batch norm\n",
        "        self.use_batch_norm = use_batch_norm\n",
        "\n",
        "        # define hidden linear layers\n",
        "        # layers with batch_norm -> no bias term\n",
        "        ## Layer 1\n",
        "        # use batch norm\n",
        "        if use_batch_norm:\n",
        "            self.fc1 = nn.Linear(input_size, hidden_dim*2, bias=False)\n",
        "            self.batch_norm1 = nn.BatchNorm1d(hidden_dim*2)\n",
        "        # do not use batch norm\n",
        "        else:\n",
        "            self.fc1 = nn.Linear(input_size, hidden_dim*2)\n",
        "\n",
        "        ## Layer 2\n",
        "        # use batch norm\n",
        "        if use_batch_norm:\n",
        "            self.fc2 = nn.Linear(hidden_dim*2, hidden_dim, bias=False)\n",
        "            self.batch_norm2 = nn.BatchNorm1d(hidden_dim)\n",
        "        # do not use batch norm\n",
        "        else:\n",
        "            self.fc2 = nn.Linear(hidden_dim*2, hidden_dim)\n",
        "\n",
        "        ## Layer 3 (output layer)\n",
        "        self.fc3 = nn.Linear(hidden_dim, output_size)\n",
        "\n",
        "\n",
        "    def forward(self, x):\n",
        "        # flatten image (reshape)\n",
        "        x = x.view(-1, 28*28)\n",
        "\n",
        "        ## Layer 1\n",
        "        # linear\n",
        "        x = self.fc1(x)\n",
        "        # batch norm\n",
        "        if self.use_batch_norm:\n",
        "            x = self.batch_norm1(x)\n",
        "        # activation function\n",
        "        x = F.relu(x)\n",
        "\n",
        "        ## Layer 2\n",
        "        x = self.fc2(x)\n",
        "        if self.use_batch_norm:\n",
        "            x = self.batch_norm2(x)\n",
        "        x = F.relu(x)\n",
        "\n",
        "        ## Layer 3: linear (no batch norm, activation function)\n",
        "        x = self.fc3(x)\n",
        "\n",
        "        return x\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QlsttRxTA9_O"
      },
      "source": [
        "### Create two different models\n",
        "\n",
        "* `net_batchnorm`: **with** batch normalization\n",
        "* `net_no_norm`: **without** batch normalization (a plain MLP)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "19qgkgeyA9_Q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c2b5453a-27dc-4f55-8c33-91b5688c04b1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "NeuralNet(\n",
            "  (fc1): Linear(in_features=784, out_features=512, bias=False)\n",
            "  (batch_norm1): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (fc2): Linear(in_features=512, out_features=256, bias=False)\n",
            "  (batch_norm2): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (fc3): Linear(in_features=256, out_features=10, bias=True)\n",
            ")\n",
            "\n",
            "NeuralNet(\n",
            "  (fc1): Linear(in_features=784, out_features=512, bias=True)\n",
            "  (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
            "  (fc3): Linear(in_features=256, out_features=10, bias=True)\n",
            ")\n"
          ]
        }
      ],
      "source": [
        "net_batchnorm = NeuralNet(use_batch_norm=True)\n",
        "net_no_norm = NeuralNet(use_batch_norm=False)\n",
        "\n",
        "# print models\n",
        "print(net_batchnorm)\n",
        "print()\n",
        "print(net_no_norm)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": true,
        "id": "Iba2BqGMA9_R"
      },
      "source": [
        "## Train\n",
        "- cross entropy loss\n",
        "- stochastic gradient descent(SGD) for optimization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "7Kqb_o96A9_S"
      },
      "outputs": [],
      "source": [
        "# train function\n",
        "def train(model, n_epochs=10):\n",
        "    # number of epochs to train model (default=10)\n",
        "    n_epochs = n_epochs\n",
        "\n",
        "    # track loss\n",
        "    losses = []\n",
        "\n",
        "    # loss function\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "    # optimizer (SGD)\n",
        "    optimizer = torch.optim.SGD(model.parameters(), lr=0.01)\n",
        "\n",
        "    # set the model to training mode\n",
        "    model.train()\n",
        "\n",
        "    for epoch in range(1, n_epochs+1):\n",
        "        # monitor training loss\n",
        "        train_loss = 0.0\n",
        "\n",
        "        ###################\n",
        "        # train the model #\n",
        "        ###################\n",
        "        batch_count = 0\n",
        "        for batch_idx, (data, target) in enumerate(train_loader):\n",
        "            # clear the gradients\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            # forward\n",
        "            output = model(data)\n",
        "\n",
        "            # loss\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # backward\n",
        "            loss.backward()\n",
        "            # update parameter: a single optimization step\n",
        "            optimizer.step()\n",
        "\n",
        "            # update train loss\n",
        "            # loss.item(): get scalar loss value of a single batch\n",
        "            train_loss += loss.item()\n",
        "            batch_count +=1\n",
        "\n",
        "        # loss (average batch loss)\n",
        "        losses.append(train_loss/batch_count)\n",
        "        print('Epoch: {} \\tTraining Loss: {:.6f}'.format(epoch, train_loss/batch_count))\n",
        "\n",
        "    # return all recorded batch losses\n",
        "    return losses"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xnmZnK-_A9_S"
      },
      "source": [
        "### Compare models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FPAKeqRnA9_T",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eb091c8b-c5f3-4358-fcfe-f2014afadc2f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 \tTraining Loss: 0.393225\n",
            "Epoch: 2 \tTraining Loss: 0.163401\n",
            "Epoch: 3 \tTraining Loss: 0.113395\n",
            "Epoch: 4 \tTraining Loss: 0.084088\n",
            "Epoch: 5 \tTraining Loss: 0.064032\n",
            "Epoch: 6 \tTraining Loss: 0.049439\n",
            "Epoch: 7 \tTraining Loss: 0.038569\n",
            "Epoch: 8 \tTraining Loss: 0.030373\n",
            "Epoch: 9 \tTraining Loss: 0.024097\n",
            "Epoch: 10 \tTraining Loss: 0.019302\n"
          ]
        }
      ],
      "source": [
        "# losses (with batch norm)\n",
        "losses_batchnorm = train(net_batchnorm)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BFvuu8-EA9_T",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "21331690-4a0e-4b6d-9ca8-6cf476877976"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 \tTraining Loss: 1.557275\n",
            "Epoch: 2 \tTraining Loss: 0.495918\n",
            "Epoch: 3 \tTraining Loss: 0.372220\n",
            "Epoch: 4 \tTraining Loss: 0.327937\n",
            "Epoch: 5 \tTraining Loss: 0.300059\n",
            "Epoch: 6 \tTraining Loss: 0.278639\n",
            "Epoch: 7 \tTraining Loss: 0.260513\n",
            "Epoch: 8 \tTraining Loss: 0.244380\n",
            "Epoch: 9 \tTraining Loss: 0.229659\n",
            "Epoch: 10 \tTraining Loss: 0.215995\n"
          ]
        }
      ],
      "source": [
        "# losses (without batch norm)\n",
        "losses_no_norm = train(net_no_norm)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cn7f12ulA9_U",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 409
        },
        "outputId": "84c665b6-0114-4236-a29c-57ff31abb4ad"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7ca2ade17df0>"
            ]
          },
          "metadata": {},
          "execution_count": 28
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 800x400 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAp8AAAF2CAYAAAAoS/PfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABREklEQVR4nO3deXhTZd4+8PucrF1D6UbBQmvZQaGyVGAG8aXKMMgMjPOKuLDNuCIudZyhDpuoFETmhwqC8Kq4IagDjKKgWEUUcdisCzuyCqQLtE3XJM05vz9OkzZdaFOanKS5P9d1riRPzvJtq849z3me5wiyLMsgIiIiIvIBUe0CiIiIiCh4MHwSERERkc8wfBIRERGRzzB8EhEREZHPMHwSERERkc8wfBIRERGRzzB8EhEREZHPMHwSERERkc8wfBIRERGRzzB8ElFQmDJlCpKSklp07Lx58yAIQusWREQUpBg+iUhVgiA0a9u+fbvapapiypQpCA8PV7sMIqJWI/DZ7kSkprffftvt85tvvolt27bhrbfecmu/6aabEB8f3+Lr2O12SJIEg8Hg8bFVVVWoqqqC0Whs8fVbasqUKfjggw9QWlrq82sTEXmDVu0CiCi43XXXXW6fv/vuO2zbtq1ee13l5eUIDQ1t9nV0Ol2L6gMArVYLrZb/uSQiag287U5Efm/EiBHo27cv9u3bh+HDhyM0NBRPPvkkAOA///kPxowZg44dO8JgMCAlJQVPP/00HA6H2znqjvk8deoUBEHA888/j1WrViElJQUGgwGDBg3Cnj173I5taMynIAh46KGHsGnTJvTt2xcGgwF9+vTB1q1b69W/fft2DBw4EEajESkpKXjllVdafRzp+++/jwEDBiAkJAQxMTG46667cO7cObd9zGYzpk6diquuugoGgwEJCQn44x//iFOnTrn22bt3L0aNGoWYmBiEhIQgOTkZ06ZNczuPJElYunQp+vTpA6PRiPj4eNx3330oLCx026855yKi4MP/K09EAeHixYsYPXo0br/9dtx1112uW/Br1qxBeHg4MjIyEB4eji+++AJz5syBxWLB4sWLmzzv2rVrUVJSgvvuuw+CIOC5557Dn/70J5w4caLJ3tJvvvkGGzZswIMPPoiIiAi8+OKLuPXWW3HmzBlER0cDAL7//nv87ne/Q0JCAp566ik4HA7Mnz8fsbGxV/5LqbZmzRpMnToVgwYNQlZWFnJzc/HCCy9g586d+P7779GuXTsAwK233ooDBw5gxowZSEpKQl5eHrZt24YzZ864Pt98882IjY3FzJkz0a5dO5w6dQobNmxwu959993nuubDDz+MkydPYtmyZfj++++xc+dO6HS6Zp+LiIKQTETkR6ZPny7X/U/TDTfcIAOQV65cWW//8vLyem333XefHBoaKldWVrraJk+eLHfp0sX1+eTJkzIAOTo6Wr506ZKr/T//+Y8MQP7oo49cbXPnzq1XEwBZr9fLx48fd7X98MMPMgD5pZdecrWNHTtWDg0Nlc+dO+dqO3bsmKzVauudsyGTJ0+Ww8LCGv3eZrPJcXFxct++feWKigpX++bNm2UA8pw5c2RZluXCwkIZgLx48eJGz7Vx40YZgLxnz55G9/n6669lAPI777zj1r5161a39uaci4iCE2+7E1FAMBgMmDp1ar32kJAQ1/uSkhIUFBTgt7/9LcrLy3H48OEmzzthwgRERUW5Pv/2t78FAJw4caLJY9PT05GSkuL6fO211yIyMtJ1rMPhwOeff45x48ahY8eOrv26du2K0aNHN3n+5ti7dy/y8vLw4IMPuk2IGjNmDHr27ImPP/4YgPJ70uv12L59e73b407OHtLNmzfDbrc3uM/7778Pk8mEm266CQUFBa5twIABCA8Px5dfftnscxFRcGL4JKKA0KlTJ+j1+nrtBw4cwPjx42EymRAZGYnY2FjXZKXi4uImz9u5c2e3z84g2lhAu9yxzuOdx+bl5aGiogJdu3att19DbS1x+vRpAECPHj3qfdezZ0/X9waDAYsWLcKWLVsQHx+P4cOH47nnnoPZbHbtf8MNN+DWW2/FU089hZiYGPzxj3/E66+/DqvV6trn2LFjKC4uRlxcHGJjY9220tJS5OXlNftcRBScOOaTiAJC7R5Op6KiItxwww2IjIzE/PnzkZKSAqPRiP379+Mf//gHJElq8rwajabBdrkZq9BdybFqePTRRzF27Fhs2rQJn376KWbPno2srCx88cUXSE1NhSAI+OCDD/Ddd9/ho48+wqeffopp06ZhyZIl+O677xAeHg5JkhAXF4d33nmnwWs4x7I251xEFJzY80lEAWv79u24ePEi1qxZg0ceeQS33HIL0tPT3W6jqykuLg5GoxHHjx+v911DbS3RpUsXAMCRI0fqfXfkyBHX904pKSl4/PHH8dlnn+Hnn3+GzWbDkiVL3Pa5/vrr8eyzz2Lv3r145513cODAAaxbt851/MWLFzFs2DCkp6fX2/r169fscxFRcGL4JKKA5ex5rN3TaLPZ8PLLL6tVkhuNRoP09HRs2rQJ58+fd7UfP34cW7ZsaZVrDBw4EHFxcVi5cqXbLe0tW7bg0KFDGDNmDABlXdTKykq3Y1NSUhAREeE6rrCwsF6vbf/+/QHAtc9tt90Gh8OBp59+ul4tVVVVKCoqava5iCg48bY7EQWsoUOHIioqCpMnT8bDDz8MQRDw1ltv+dVt73nz5uGzzz7DsGHD8MADD8DhcGDZsmXo27cvcnJymnUOu92OZ555pl57+/bt8eCDD2LRokWYOnUqbrjhBkycONG11FJSUhIee+wxAMDRo0cxcuRI3Hbbbejduze0Wi02btyI3Nxc3H777QCAN954Ay+//DLGjx+PlJQUlJSUYPXq1YiMjMTvf/97AMpYzvvuuw9ZWVnIycnBzTffDJ1Oh2PHjuH999/HCy+8gD//+c/NOhcRBSeGTyIKWNHR0di8eTMef/xxzJo1C1FRUbjrrrswcuRIjBo1Su3yAAADBgzAli1b8Le//Q2zZ89GYmIi5s+fj0OHDjVrNj6g9ObOnj27XntKSgoefPBBTJkyBaGhoVi4cCH+8Y9/ICwsDOPHj8eiRYtcs84TExMxceJEZGdn46233oJWq0XPnj3x3nvv4dZbbwWgBMvdu3dj3bp1yM3NhclkwuDBg/HOO+8gOTnZdd2VK1diwIABeOWVV/Dkk09Cq9UiKSkJd911F4YNG+bRuYgo+PDZ7kREKhg3bhwOHDiAY8eOqV0KEZFPccwnEZGXVVRUuH0+duwYPvnkE4wYMUKdgoiIVMSeTyIiL0tISMCUKVNw9dVX4/Tp01ixYgWsViu+//57dOvWTe3yiIh8imM+iYi87He/+x3effddmM1mGAwGDBkyBAsWLGDwJKKgxJ5PIiIiIvIZjvkkIiIiIp9h+CQiIiIinwmIMZ+SJOH8+fOIiIiAIAhql0NEREREdciyjJKSEnTs2BGi2Hj/ZkCEz/PnzyMxMVHtMoiIiIioCWfPnsVVV13V6PcBET4jIiIAKD9MZGSkytUQERERUV0WiwWJiYmu3NYYj8Pnjh07sHjxYuzbtw8XLlzAxo0bMW7cuMseY7VaMX/+fLz99tswm81ISEjAnDlzMG3atGZd03mrPTIykuGTiIiIyI81NUTS4/BZVlaGfv36Ydq0afjTn/7UrGNuu+025Obm4tVXX0XXrl1x4cIFSJLk6aWJiIiIKMB5HD5Hjx6N0aNHN3v/rVu34quvvsKJEyfQvn17AEBSUpKnlyUiIiKiNsDrSy19+OGHGDhwIJ577jl06tQJ3bt3x9/+9rd6zzquzWq1wmKxuG1EREREFPi8PuHoxIkT+Oabb2A0GrFx40YUFBTgwQcfxMWLF/H66683eExWVhaeeuopb5dGRETktxwOB+x2u9plELnodDpoNJorPs8VPV5TEIQmJxzdfPPN+Prrr2E2m2EymQAAGzZswJ///GeUlZUhJCSk3jFWqxVWq9X12Tl7qri4mBOOiIioTZNlGWazGUVFRWqXQlRPu3bt0KFDhwYnFVksFphMpibzmtd7PhMSEtCpUydX8ASAXr16QZZl/Prrr+jWrVu9YwwGAwwGg7dLIyIi8jvO4BkXF4fQ0FA+XIX8gizLKC8vR15eHgAl37WU18PnsGHD8P7776O0tBTh4eEAgKNHj0IUxcsuQEpERBRsHA6HK3hGR0erXQ6RG+fd6ry8PMTFxbX4FrzHE45KS0uRk5ODnJwcAMDJkyeRk5ODM2fOAAAyMzMxadIk1/533HEHoqOjMXXqVBw8eBA7duzAE088gWnTpjV4y52IiChYOcd4hoaGqlwJUcOc/2xeyXhkj8Pn3r17kZqaitTUVABARkYGUlNTMWfOHADAhQsXXEEUAMLDw7Ft2zYUFRVh4MCBuPPOOzF27Fi8+OKLLS6aiIioLeOtdvJXrfHPpse33UeMGIHLzVFas2ZNvbaePXti27Ztnl5KXbIM8F9+IiIiolbl9XU+A461BDj4H2Dva0oAJSIiIp+YN28e+vfv7/XrTJkypclHg3uLIAjYtGmTKtf2FwyfdWmNQMFRoDQPKM1VuxoiIqKAMGLECDz66KP12tesWYN27do16xx/+9vfkJ2d3bqFecmpU6cgCIJrDgw1H8NnXRod0D5FeZ9/WN1aiIiIgkh4eDhn+XuJPz2wgOGzIbE9lNf8o7z1TkRE1Iq2b9+OwYMHIywsDO3atcOwYcNw+vRpAPVvuztvjz///PNISEhAdHQ0pk+f7hakLly4gDFjxiAkJATJyclYu3YtkpKSsHTp0iZreeqppxAbG4vIyEjcf//9sNlsru+2bt2K3/zmN2jXrh2io6Nxyy234JdffnF9n5ycDABITU2FIAgYMWKE67vXXnsNffr0gcFgQEJCAh566CG36xYUFGD8+PEIDQ1Ft27d8OGHH7r9fgRBQHZ2NgYOHIjQ0FAMHToUR44ccTvHihUrkJKSAr1ejx49euCtt95y+14QBKxYsQJ/+MMfEBYWhmeffdb1+33ttdfQuXNnhIeH48EHH4TD4cBzzz2HDh06IC4uDs8++2yTv7srwfDZkOiugKgByi8CZQVqV0NEREFMlmXYqiSfb1fwAMRGVVVVYdy4cbjhhhvw448/YteuXbj33nsvO4P6yy+/xC+//IIvv/wSb7zxBtasWeM2uXnSpEk4f/48tm/fjn//+99YtWqVayH0y8nOzsahQ4ewfft2vPvuu9iwYYPbo73LysqQkZGBvXv3Ijs7G6IoYvz48ZAkCQCwe/duAMDnn3+OCxcuYMOGDQCUUDh9+nTce++9+Omnn/Dhhx+ia9eubtd+6qmncNttt+HHH3/E73//e9x55524dOmS2z7//Oc/sWTJEuzduxdarRbTpk1zfbdx40Y88sgjePzxx/Hzzz/jvvvuw9SpU/Hll1+6nWPevHkYP348fvrpJ9fxv/zyC7Zs2YKtW7fi3XffxauvvooxY8bg119/xVdffYVFixZh1qxZ+O9//9vk77ClvL7IfEDSGoCoZODicaDgCBAeq3ZFREQUpOwOGcu/PO7z606/sSv02tZd9cVisaC4uBi33HILUlKUIW69evW67DFRUVFYtmwZNBoNevbsiTFjxiA7Oxv33HMPDh8+jM8//xx79uzBwIEDAQD/93//1+DTE+vS6/V47bXXEBoaij59+mD+/Pl44okn8PTTT0MURdx6661u+7/22muIjY3FwYMH0bdvX8TGKtkgOjoaHTp0cO33zDPP4PHHH8cjjzziahs0aJDbuaZMmYKJEycCABYsWIAXX3wRu3fvxu9+9zvXPs8++yxuuOEGAMDMmTMxZswYVFZWwmg04vnnn8eUKVPw4IMPAlCWvfzuu+/w/PPP48Ybb3Sd44477sDUqVPdri1JEl577TVERESgd+/euPHGG3HkyBF88sknEEURPXr0wKJFi/Dll18iLS2tyd9jS7DnszGuW+8c90lERNQa2rdvjylTpmDUqFEYO3YsXnjhBVy4cOGyx/Tp08ftSToJCQmuns0jR45Aq9Xiuuuuc33ftWtXREVFNVlLv3793BbzHzJkCEpLS3H27FkAwLFjxzBx4kRcffXViIyMRFJSEgC4rWVeV15eHs6fP4+RI0de9trXXnut631YWBgiIyPr9dbW3sf5KEvnPocOHcKwYcPc9h82bBgOHTrk1uYM5LUlJSUhIiLC9Tk+Ph69e/eGKIpubc3pPW4p9nw2JqYbIIhAaT5QfgkIba92RUREFIR0GgHTb+za9I5euK4nIiMjUVxcXK+9qKgIJpPJ9fn111/Hww8/jK1bt2L9+vWYNWsWtm3bhuuvv77hOnQ6t8+CILhufXvT2LFj0aVLF6xevRodO3aEJEno27ev27jQupr75Mbm/Ey193EOS/D05w4LC2vWtX39O2bPZ2N0IUBUF+V9/pHL70tEROQlgiBArxV9vnn6JJsePXpg//799dr379+P7t27u7WlpqYiMzMT3377Lfr27Yu1a9e26HfTo0cPVFVV4fvvv3e1HT9+HIWFhU0e+8MPP6CiosL1+bvvvkN4eDgSExNx8eJFHDlyBLNmzcLIkSPRq1eveufU6/UAAIfD4WqLiIhAUlKS15eL6tWrF3bu3OnWtnPnTvTu3dur120t7Pm8nJjuwKWTyrjPLkPUroaIiMhvPfDAA1i2bBkefvhh/PWvf4XBYMDHH3+Md999Fx999BEA4OTJk1i1ahX+8Ic/oGPHjjhy5AiOHTuGSZMmteiaPXv2RHp6Ou69916sWLECOp0Ojz/+OEJCQpoMzzabDX/5y18wa9YsnDp1CnPnzsVDDz0EURQRFRWF6OhorFq1CgkJCThz5gxmzpzpdnxcXBxCQkKwdetWXHXVVTAajTCZTJg3bx7uv/9+xMXFYfTo0SgpKcHOnTsxY8aMFv2MDXniiSdw2223ITU1Fenp6fjoo4+wYcMGfP755612DW9iz+flxHRXHrFpuQBU1r+VQERERIqrr74aO3bswOHDh5Geno60tDS89957eP/9910TaUJDQ3H48GHceuut6N69O+69915Mnz4d9913X4uv++abbyI+Ph7Dhw/H+PHjcc899yAiIgJGo/Gyx40cORLdunXD8OHDMWHCBPzhD3/AvHnzAACiKGLdunXYt28f+vbti8ceewyLFy92O16r1eLFF1/EK6+8go4dO+KPf/wjAGDy5MlYunQpXn75ZfTp0we33HILjh071uKfryHjxo3DCy+8gOeffx59+vTBK6+8gtdff91tuSd/JsjeWEuhlVksFphMJhQXFyMyMtK3F//+baDoLNA1HUgc1PT+RERELVRZWYmTJ08iOTm5yfBEDfv111+RmJiIzz//vMmJP+S5y/0z2ty8xtvuTYntqYTP/MMMn0RERH7miy++QGlpKa655hpcuHABf//735GUlIThw4erXRo1grfdmxJTvVaY5RxgLVW3FiIiInJjt9vx5JNPok+fPhg/fjxiY2Oxffv2ejO4yX+w57MpRhMQmaCM+yw4AnQaoHZFREREVG3UqFEYNWqU2mWQB9jz2RyxPZXX/KPq1kFEREQU4Bg+myOmen2yojOArVzdWoiIiIgCGMNnc4S2B8LjAFkCLrbucglEREREwYThs7lcz3rn046IiIiIWorhs7mc4z4LTwH2SlVLISIiIgpUDJ/NFRYDhEYDkgO4eFztaoiIiIgCEsOnJ5y33gt4652IiIioJRg+PeEMn5dOAFU2dWshIiLyI1OmTIEgCFi4cKFb+6ZNmyAIgkpVkT9i+PREeDwQ0g5wVCkBlIiIiFyMRiMWLVqEwsJCtUtpkizLqKqqUruMoMTw6QlBqFnzM/+wurUQERH5mfT0dHTo0AFZWVmX3e/f//43+vTpA4PBgKSkJCxZsuSy+8+bNw/9+/fHW2+9haSkJJhMJtx+++0oKSlx7WO1WvHwww8jLi4ORqMRv/nNb7Bnzx7X99u3b4cgCNiyZQsGDBgAg8GAb775BiNGjMCMGTPw6KOPIioqCvHx8Vi9ejXKysowdepUREREoGvXrtiyZcuV/XLIxePwuWPHDowdOxYdO3aEIAjYtGlTs4/duXMntFot+vfv7+ll/Ydz1vvF40oPKBERkTfJsjLUy9ebLHtcqkajwYIFC/DSSy/h119/bXCfffv24bbbbsPtt9+On376CfPmzcPs2bOxZs2ay577l19+waZNm7B582Zs3rwZX331ldst/r///e/497//jTfeeAP79+9H165dMWrUKFy6dMntPDNnzsTChQtx6NAhXHvttQCAN954AzExMdi9ezdmzJiBBx54AP/7v/+LoUOHYv/+/bj55ptx9913o7ycD5ppDR4/272srAz9+vXDtGnT8Kc//anZxxUVFWHSpEkYOXIkcnNzPb2s/4jsCBgiAGsJUHgSiOmmdkVERNSWOezA15fvGfSK3z4OaPUeHzZ+/Hj0798fc+fOxauvvlrv+3/9618YOXIkZs+eDQDo3r07Dh48iMWLF2PKlCmNnleSJKxZswYREREAgLvvvhvZ2dl49tlnUVZWhhUrVmDNmjUYPXo0AGD16tXYtm0bXn31VTzxxBOu88yfPx833XST27n79euHWbNmAQAyMzOxcOFCxMTE4J577gEAzJkzBytWrMCPP/6I66+/3uPfCbnzuOdz9OjReOaZZzB+/HiPjrv//vtxxx13YMiQIZ5e0r8IAhecJyIiuoxFixbhjTfewKFDh+p9d+jQIQwbNsytbdiwYTh27BgcDkej50xKSnIFTwBISEhAXl4eAKVX1G63u51Xp9Nh8ODB9WoYOHBgvXM7e0ABpfc2Ojoa11xzjastPj4eAFzXoyvjcc9nS7z++us4ceIE3n77bTzzzDO+uKR3xXQHft2rPGpTcgCiRu2KiIiordLolF5INa7bQsOHD8eoUaOQmZl52d5MT+h07vUIggBJkjw+T1hYWLPOXbvNOVu/Jdej+rwePo8dO4aZM2fi66+/hlbbvMtZrVZYrVbXZ4vF4q3yWsaUCOhDAVs5UHQaaH+12hUREVFbJQgtuv2ttoULF6J///7o0aOHW3uvXr2wc+dOt7adO3eie/fu0Gha1pmTkpICvV6PnTt3okuXLgAAu92OPXv24NFHH23ROcl7vDrb3eFw4I477sBTTz2F7t27N/u4rKwsmEwm15aYmOjFKltAFGvNej+qbi1ERER+6JprrsGdd96JF1980a398ccfR3Z2Np5++mkcPXoUb7zxBpYtW4a//e1vLb5WWFgYHnjgATzxxBPYunUrDh48iHvuuQfl5eX4y1/+cqU/CrUyr4bPkpIS7N27Fw899BC0Wi20Wi3mz5+PH374AVqtFl988UWDx2VmZqK4uNi1nT171ptltkztpx2xG56IiKie+fPn17tVfd111+G9997DunXr0LdvX8yZMwfz58+/4tvzCxcuxK233oq7774b1113HY4fP45PP/0UUVFRV3Rean2CLLdgLQXnwYKAjRs3Yty4cQ1+L0kSDh486Nb28ssv44svvsAHH3yA5OTkBsde1GWxWGAymVBcXIzIyMiWltu6JAfw7YuAvRJIvRNo11ntioiIKMBVVlbi5MmTSE5OhtFoVLsconou989oc/Oax2M+S0tLcfz4cdfnkydPIicnB+3bt0fnzp2RmZmJc+fO4c0334Qoiujbt6/b8c7FX+u2BxxRA0R3A8w/KbPeGT6JiIiImuTxbfe9e/ciNTUVqampAICMjAykpqZizpw5AIALFy7gzJkzrVulv6q95FLLO5CJiIiIgsYV3Xb3Fb+87Q4oTzjauVRZAPi6SYCpk9oVERFRAONtd/J3rXHbnc92vxIaLRDdVXlfwAXniYiIiJrC8HmlnM965613IiIioiYxfF6p9lcrPaAVRUApH7tFRERXjk/SIX/VGv9s+uTxmm2aVq8E0PyjQP5hICJe7YqIiChA6fV6iKKI8+fPIzY2Fnq93vVoRyI1ybIMm82G/Px8iKIIvb7lT91i+GwNMT2U8FlwFLj6BrWrISKiACWKIpKTk3HhwgWcP39e7XKI6gkNDUXnzp0hii2/ec7w2RqiuyrrfpYVKFtYjNoVERFRgNLr9ejcuTOqqqrgcDjULofIRaPRQKvVXnFvPMNna9AZgagk4OIvysQjhk8iIroCgiBAp9NBp9OpXQpRq+OEo9biWnD+sLp1EBEREfkxhs/WEt0NEERlxntFodrVEBEREfklhs/Wog+teb57PhecJyIiImoIw2driu2uvDJ8EhERETWI4bM1xXQHBAGwnAcqLWpXQ0REROR3GD5bkyECiOykvC84qm4tRERERH6I4bO1cdY7ERERUaMYPltbTPW4z+JfAWupurUQERER+RmGz9YW0g6I6ADIMm+9ExEREdXB8OkNsT2VV4ZPIiIiIjcMn97gHPdZeBqwV6hbCxEREZEfYfj0htD2QHgsIEtAwTG1qyEiIiLyGwyf3hLjnPXOBeeJiIiInBg+vcU57rPwJFBlVbcWIiIiIj/B8OktYTHK7XfJAVw8rnY1RERERH6B4dNbBKHWgvO89U5EREQEMHx6l3Pc56VfAIdd3VqIiIiI/ADDpzdFdACMJsBRBVw6oXY1RERERKpj+PQmQQBiqx+3yWe9ExEREXkePnfs2IGxY8eiY8eOEAQBmzZtuuz+GzZswE033YTY2FhERkZiyJAh+PTTT1tab+Bxznq/eFzpASUiIiIKYh6Hz7KyMvTr1w/Lly9v1v47duzATTfdhE8++QT79u3DjTfeiLFjx+L777/3uNiAFNkJMIQDVTag6LTa1RARERGpSpBlWW7xwYKAjRs3Yty4cR4d16dPH0yYMAFz5sxp1v4WiwUmkwnFxcWIjIxsQaUqO/oZcG4fkHAt0HOM2tUQERERtbrm5jWtD2sCAEiShJKSErRv377RfaxWK6zWmoXZLRaLL0rzntjuSvgsOAZIEiByqC0REREFJ5+noOeffx6lpaW47bbbGt0nKysLJpPJtSUmJvqwQi8wdQZ0IYC9grfeiYiIKKj5NHyuXbsWTz31FN577z3ExcU1ul9mZiaKi4td29mzZ31YpReIIhBTPeu94Ki6tRARERGpyGfhc926dfjrX/+K9957D+np6Zfd12AwIDIy0m0LeLWfdtTyYbZEREREAc0n4fPdd9/F1KlT8e6772LMmCCdcBOVBGgNgK0MKP5V7WqIiIiIVOFx+CwtLUVOTg5ycnIAACdPnkROTg7OnDkDQLllPmnSJNf+a9euxaRJk7BkyRKkpaXBbDbDbDajuLi4dX6CQCFqgJhuyns+652IiIiClMfhc+/evUhNTUVqaioAICMjA6mpqa5lky5cuOAKogCwatUqVFVVYfr06UhISHBtjzzySCv9CAHE+az3At56JyIiouB0Ret8+krAr/Pp5LADO19QXgdMBiI7ql0RERERUatobl7jgpO+pNEB0SnKe956JyIioiDE8Olrzme9c9Y7ERERBSGGT19rnwKIWqCiECjLV7saIiIiIp9i+PQ1rR5on6y8zz+sbi1EREREPsbwqYbaC84TERERBRGGTzVEdwUEESgrAMouql0NERERkc8wfKpBF6I88QhQ1vwkIiIiChIMn2qJ7a68ctwnERERBRGGT7XEdAcEASjJVWa+ExEREQUBhk+16MMAU6LyPv+ourUQERER+QjDp5qcC85z3CcREREFCYZPNcV0U16LzwGVFnVrISIiIvIBhk81GSMBUyflfcExdWshIiIi8gGGT7XFOBec56x3IiIiavsYPtXmXHKp+CxgK1O3FiIiIiIvY/hUW0gUEBEPyDJQwFnvRERE1LYxfPoD56x3LrlEREREbRzDpz9wjvssPAXYK1QthYiIiMibGD79QVg0EBYDyBJw8bja1RARERF5DcOnv4h1znrngvNERETUdjF8+gvnuM9LJ4Eqq7q1EBEREXkJw6e/CItVZr5LVcClE2pXQ0REROQVDJ/+QhBq3XrngvNERETUNjF8+hNn+Lz4C+Cwq1sLERERkRcwfPqTiATlee8OuzL2k4iIiKiN8Th87tixA2PHjkXHjh0hCAI2bdrU5DHbt2/HddddB4PBgK5du2LNmjUtKDUICELNmp8FnPVOREREbY/H4bOsrAz9+vXD8uXLm7X/yZMnMWbMGNx4443IycnBo48+ir/+9a/49NNPPS42KDhvvRccAySHurUQERERtTKtpweMHj0ao0ePbvb+K1euRHJyMpYsWQIA6NWrF7755hv8v//3/zBq1ChPL9/2ma4C9GGArUx54lF0itoVEREREbUar4/53LVrF9LT093aRo0ahV27dnn70oHJbdY7b70TERFR2+L18Gk2mxEfH+/WFh8fD4vFgoqKhp9jbrVaYbFY3LagEtNdeS04CkiSurUQERERtSK/nO2elZUFk8nk2hITE9UuybfadQF0IYC9Aig+o3Y1RERERK3G6+GzQ4cOyM3NdWvLzc1FZGQkQkJCGjwmMzMTxcXFru3s2bPeLtO/iCIQ0015n39U3VqIiIiIWpHXw+eQIUOQnZ3t1rZt2zYMGTKk0WMMBgMiIyPdtqDjfNZ7wRFAltWthYiIiKiVeBw+S0tLkZOTg5ycHADKUko5OTk4c0a5PZyZmYlJkya59r///vtx4sQJ/P3vf8fhw4fx8ssv47333sNjjz3WOj9BW9WuC6DVA9ZSwHJO7WqIiIiIWoXH4XPv3r1ITU1FamoqACAjIwOpqamYM2cOAODChQuuIAoAycnJ+Pjjj7Ft2zb069cPS5Yswf/93/9xmaWmaLRAtPPWO5/1TkRERG2DIMv+f0/XYrHAZDKhuLg4uG7B5x8Bft4AGE3A9Q8oyzARERER+aHm5jW/nO1O1dpfrfSAVhYDJWa1qyEiIiK6Ygyf/kyjA9pXP+GIz3onIiKiNoDh09/VftqR/4+QICIiIroshk9/F90VEDVA+SWgrEDtaoiIiIiuCMOnv9MagKhk5T1nvRMREVGAY/gMBM5b7xz3SURERAGO4TMQxHQDBBEozVduvxMREREFKIbPQKALAaK6KO/z2ftJREREgYvhM1DEdFdeOe6TiIiIAhjDZ6CI6a484ajEDFQUqV0NERERUYswfAYKQzhgukp5X3BU3VqIiIiIWojhM5DE9lReOe6TiIiIAhTDZyBxjvu0nAOsJerWQkRERNQCDJ+BxBgJRHZUHrPJW+9EREQUgBg+A03tZ70TERERBRiGz0DjDJ9FZwFbubq1EBEREXmI4TPQhEQB4XGALPHWOxEREQUchs9A5Jz1zvBJREREAYbhMxA5b70XngLslaqWQkREROQJhs9AFBajbJIDuHhc7WqIiIiImo3hM1A51/ws4Kx3IiIiChwMn4HKOe7z0gmgyqZuLURERETNxPAZqMLjgJB2gKNKCaBEREREAYDhM1AJQq0F5w+rWwsRERFRMzF8BrKY6vB58bjSA0pERETk5xg+A1lkR8AQATjsQOFJtashIiIialKLwufy5cuRlJQEo9GItLQ07N69+7L7L126FD169EBISAgSExPx2GOPobKS61NeMbdb75z1TkRERP7P4/C5fv16ZGRkYO7cudi/fz/69euHUaNGIS8vr8H9165di5kzZ2Lu3Lk4dOgQXn31Vaxfvx5PPvnkFRdPqAmfF48p634SERER+TGPw+e//vUv3HPPPZg6dSp69+6NlStXIjQ0FK+99lqD+3/77bcYNmwY7rjjDiQlJeHmm2/GxIkTm+wtpWaKvArQhypPOio6rXY1RERERJflUfi02WzYt28f0tPTa04gikhPT8euXbsaPGbo0KHYt2+fK2yeOHECn3zyCX7/+983eh2r1QqLxeK2USNEsWbiEW+9ExERkZ/zKHwWFBTA4XAgPj7erT0+Ph5ms7nBY+644w7Mnz8fv/nNb6DT6ZCSkoIRI0Zc9rZ7VlYWTCaTa0tMTPSkzOAT63za0VFAktSthYiIiOgyvD7bffv27ViwYAFefvll7N+/Hxs2bMDHH3+Mp59+utFjMjMzUVxc7NrOnj3r7TIDW7sugM4I2MqBYv6uiIiIyH9pPdk5JiYGGo0Gubm5bu25ubno0KFDg8fMnj0bd999N/76178CAK655hqUlZXh3nvvxT//+U+IYv38azAYYDAYPCktuIkaILobYP5J6f2M6qJ2RUREREQN8qjnU6/XY8CAAcjOzna1SZKE7OxsDBkypMFjysvL6wVMjUYDAJBl2dN6qTG1l1zi75WIiIj8lEc9nwCQkZGByZMnY+DAgRg8eDCWLl2KsrIyTJ06FQAwadIkdOrUCVlZWQCAsWPH4l//+hdSU1ORlpaG48ePY/bs2Rg7dqwrhFIriEoGNDrAWgJYzgOmTmpXRERERFSPx+FzwoQJyM/Px5w5c2A2m9G/f39s3brVNQnpzJkzbj2ds2bNgiAImDVrFs6dO4fY2FiMHTsWzz77bOv9FARotEB0VyDvkPKsd4ZPIiIi8kOCHAD3vi0WC0wmE4qLixEZGal2Of4r7zBwYCMQ0g5Iu195AhIRERGRDzQ3r/HZ7m1J+6uVHtCKIqA0t8ndiYiIiHyN4bMt0eqVAApwwXkiIiLySwyfbQ2fdkRERER+jOGzrYnuqqz7WX4RKCtQuxoiIiIiNwyfbY3OCEQlKe/zD6taChEREVFdDJ9tUSxvvRMREZF/Yvhsi6K7AYIIlOYB5ZfUroaIiIjIheGzLdKHAu06K+8LjqpbCxEREVEtDJ9tVWx35ZXjPomIiMiPMHy2VTE9lCccWS4AlcVqV0NEREQEgOGz7TKEA5HVz3fP5613IiIi8g8Mn21ZbE/ltYCz3omIiMg/MHy2Zc5xn8W/AtZSdWshIiIiAsNn22Y0AZEJgCxz1jsRERH5BYbPts75rHeGTyIiIvIDDJ9tnfNpR4WnAXuFurUQERFR0GP4bOtC2wPhsYAsAQXH1K6GiIiIghzDZzBwznrns96JiIhIZQyfwcA57rPwJFBlVbcWIiIiCmoMn8EgLAYIjQYkB3DxuNrVEBERURBj+AwGglDrWe+89U5ERETqYfgMFs5xn5d+ARx2dWshIiKioMXwGSzC45VF5x1VwKUTaldDREREQYrhM1gIQs2an/mH1a2FiIiIghbDZzBxhs+Lx5UeUCIiIiIfa1H4XL58OZKSkmA0GpGWlobdu3dfdv+ioiJMnz4dCQkJMBgM6N69Oz755JMWFUxXILITYAgHqmxA4Sm1qyEiIqIg5HH4XL9+PTIyMjB37lzs378f/fr1w6hRo5CXl9fg/jabDTfddBNOnTqFDz74AEeOHMHq1avRqVOnKy6ePCQItZ71zlnvRERE5HuCLMuyJwekpaVh0KBBWLZsGQBAkiQkJiZixowZmDlzZr39V65cicWLF+Pw4cPQ6XQtKtJiscBkMqG4uBiRkZEtOgdVKzwF5LwL6IzA0IcBUaN2RURERNQGNDevedTzabPZsG/fPqSnp9ecQBSRnp6OXbt2NXjMhx9+iCFDhmD69OmIj49H3759sWDBAjgcDk8uTa3F1BnQhQD2SqDojNrVEBERUZDxKHwWFBTA4XAgPj7erT0+Ph5ms7nBY06cOIEPPvgADocDn3zyCWbPno0lS5bgmWeeafQ6VqsVFovFbaNWIopADBecJyIiInV4fba7JEmIi4vDqlWrMGDAAEyYMAH//Oc/sXLlykaPycrKgslkcm2JiYneLjO4OGe9FxwFJEndWoiIiCioeBQ+Y2JioNFokJub69aem5uLDh06NHhMQkICunfvDo2mZmxhr169YDabYbPZGjwmMzMTxcXFru3s2bOelElNiUoCtAbAVgZYflW7GiIiIgoiHoVPvV6PAQMGIDs729UmSRKys7MxZMiQBo8ZNmwYjh8/DqlWD9vRo0eRkJAAvV7f4DEGgwGRkZFuG7UiUQPEdFPe5x9VtxYiIiIKKh7fds/IyMDq1avxxhtv4NChQ3jggQdQVlaGqVOnAgAmTZqEzMxM1/4PPPAALl26hEceeQRHjx7Fxx9/jAULFmD69Omt91OQ52ovueTZggdERERELab19IAJEyYgPz8fc+bMgdlsRv/+/bF161bXJKQzZ85AFGsybWJiIj799FM89thjuPbaa9GpUyc88sgj+Mc//tF6PwV5rn0yoNEBlRag5AIQ2VHtioiIiCgIeLzOpxq4zqeXHNgI5B0GOqcBKf+jdjVEREQUwLyyzie1MbE9ldf8o7z1TkRERD7B8BnM2qcAohaoKARKG348KhEREVFrYvgMZlq9MvYT4LPeiYiIyCcYPoOdc8F5Pu2IiIiIfIDhM9hFd1PW/SwrAMouql0NERERtXEMn8FOZwTadVHe5x9WtxYiIiJq8xg+qdaz3nnrnYiIiLyL4ZOUR20KAlCSq8x8JyIiIvIShk8C9GFAu87Kez7rnYiIiLyI4ZMUzme9c9wnEREReRHDJyliuyuvlvPK896JiIiIvIDhkxSGCMDUSXl//ntAcqhbDxEREbVJWrULID8S2wsoPgec/lYJoHG9gfg+QGRHZUISERER0RVi+KQanQYAtlLA/BNgKwPO7VO2kCglhMb3AULbq10lERERBTBBlmVZ7SKaYrFYYDKZUFxcjMjISLXLafskCSg8CeQeUNb+dFTVfBfZEYjvC8T1VGbJExEREaH5eY09n1SfKALRKcpWZQMKjipBtPCkMiHJch44/jnQ/mqlNzSmG6DRqV01ERERBQCGT7o8rR7o0FfZrKVA3iEg92egxAxcPK5sWr2yVFN8H+VRnSLnsREREVHDGD6p+QzhQOIgZSu7qITQ3ANAZbEyTtT8k7JPXG/l1nx4HCcqERERkRuO+aQrI8tA8a9KCM0/BNgra74Li1FCaHxvwGhSr0YiIiLyuubmNYZPaj2OKuDSCaVH9OLxmrVCBQEwJSq35WN7AjqjunUSERFRq+OEI/I9jVZ5UlJsd6UHNP+w0iNadKZmO7ZNmcgU31d5FTVqV01EREQ+xPBJ3qEzAh37K1tlMZB7UOkRLSsA8o8om86oLGwf3wcwXcXxoUREREGA4ZO8z2gCugwBOl8PlOYpITTvoDJ7/vz3yhbSrmaiUli02hUTERGRl3DMJ6lDkoCi09UTlQ4DDnvNdxEdqhey76XMniciIiK/xwlHFDgcdqDgmBJEL50AZElpF0SgfbJyWz66m7KeKBEREfklTjiiwKHRKcsxxfdWnimfdxjI/QmwXAAu/qJsGh0Q61zIPokL2RMREQWoFv0v+PLly5GUlASj0Yi0tDTs3r27WcetW7cOgiBg3LhxLbksBQN9GHDVAGDAFCDtPiBpmDIe1GEHzD8DP6wHdi1THu9ZYlbWGSUiIqKA4fFt9/Xr12PSpElYuXIl0tLSsHTpUrz//vs4cuQI4uLiGj3u1KlT+M1vfoOrr74a7du3x6ZNm5p9Td52D3KyDFjOKbfl8w4B9oqa78JilN7QuN5KSCUiIiJVeG3MZ1paGgYNGoRly5YBACRJQmJiImbMmIGZM2c2eIzD4cDw4cMxbdo0fP311ygqKmL4pJaRHDUL2RccB6Sqmu/a1V7IPkS9GomIiIKQV8Z82mw27Nu3D5mZma42URSRnp6OXbt2NXrc/PnzERcXh7/85S/4+uuvm7yO1WqF1Wp1fbZYLJ6UecUq7Q4YtCIErjvpf0QNENNN2eyVQMGRWgvZn1W22gvZt09RFr8nIiIiv+DR/yoXFBTA4XAgPj7erT0+Ph6HDx9u8JhvvvkGr776KnJycpp9naysLDz11FOelNZqZFnGhz+ch0YQkN47HqYQnSp1UDPojEBCP2WrtChrh+b+DJTmA/lHlU1rUJZsiu+jPOKT/4eCiIhIVV7tEiopKcHdd9+N1atXIyYmptnHZWZmIiMjw/XZYrEgMTHRGyXWU1BqQ56lEnaHjLe/O43fdovBNZ1M7AX1d8ZIZRH72gvZ5x4ErCXA+RxlM5qqZ9X3VcaKEhERkc95FD5jYmKg0WiQm5vr1p6bm4sOHTrU2/+XX37BqVOnMHbsWFebJClrOGq1Whw5cgQpKSn1jjMYDDAYDJ6U1mpiIwy46/ou+OxALs4VVSD7UB6O5ZayFzSQhMcB4f8DJI8Ais/ULGRfWQyc3qVsEfG1FrKPULtiIiKioNGiCUeDBw/GSy+9BEAJk507d8ZDDz1Ub8JRZWUljh8/7tY2a9YslJSU4IUXXkD37t2h1ze9cLgaE45kWcb3Z4vw7fEC2B0y9FoRw7vFom+nSPaCBiKHHbh4XAmiF3+ptZC9AEQlKbflY3pwIXsiIqIW8toi8xkZGZg8eTIGDhyIwYMHY+nSpSgrK8PUqVMBAJMmTUKnTp2QlZUFo9GIvn37uh3frl07AKjX7m8EQcB1naOQHB2GbQeVXtDPD+XiWF4J0nvHI9LIXtCAotEpvZxxvQBbOZB/SAmixeeASyeVTfhE6QUNaafcojdWvzo/68M5ZpSIiOgKeRw+J0yYgPz8fMyZMwdmsxn9+/fH1q1bXZOQzpw5A7ENPX0mKkyPPw+4ytULevpiOd7adZq9oIFMHwp0GqBs5ZeqJyodUN5XFitbQ0Stexh1hlNnm9bIcEpERNQEPtvdA4VlNnx20IzzRZUAgC7RoewFbStkWZmcVFkMVBbVhNCK6vdWS9NPU9Lq6/SWtnMPqLylT0REbZjXFplXg7+ETwCQpJqxoFWSMhb0hu6x6NORvaBtmuRQAqgzjDoDqvOzrazpc+hD69/Krx1ORY03fwIiIiKvYvj0sktlNmyr1QuaFBOKkb3YCxq0HHb3MFpZ6P65ynr54wVBGW9a91a+M6AaInhLn4iI/BrDpw8ovaCF+Pb4RfaC0uXZK+qE0yL3z7UfE9oQUVMTShvqPdWFMJwSEZGqGD596FKZDZ8dMONCcU0vaHqveESwF5SaQ5aV2/Z1A6lr7KmlZmmoxmh09ceZusKpSXnSExERkRcxfPpY3V5Qg05ZF5S9oHTFJEkZb9pYOLWWNn0OXUjDY01DogBDJKDx6sPOiIgoCDB8qqRuL2hyTBhG9opjLyh5j8Ou9I5WFimbK5xWB1R75eWPFwQlnOpCq7fq9/o6n2u3cXIUERHVwfCpIkmSsf9MIXb9wl5Q8gP2Svcw6uo9rd4cTYw3bYhWD+jClGCqr37VhdS0uYJqdZtGxzGpRERtHMOnH7hYasVnB3NhZi8o+StZBuzlyphTW5kyMcpeXr1VNNzWkv9kiNrqoFq7NzWs8TZOoCIiCjgMn36ioV7QG7rHoncCe0EpAMlydRitHUidQbW84bamZvI3hEMBiIgCDsOnn6nbC3p1bBhG9opHuIETPagNk2VlTGpTvan28prw2tSaqI2pPRSg7m3/em2hgEbP3lUiolbE8OmHJEnGvupeUEd1L+iI7nHolRDBXlAiJ8nRQFCtAOxl3hkKoDMC2upNo69+b2ji1QiIYuv/7EREAYzh048VlFrx2YFc5FrYC0p0xWQZqKps/La/q3e17MqGAtSl0dUPpE2G1lqvHCZARG0Mw6efkyQZe08X4rsT7AUl8rkqW00orapUbvVf9rXWe4e9dWq4bHhtrAe21nuGVyLyMwyfAYK9oEQBRnI0I6zWenXUbbO1Th0abQt6XBleich7GD4DSN1eUKNOgxE9YtGzA3tBidocSWogkHrQ89pa4VXUKj2sGkP1a+33huqeWUOd7/XVbXXeM8gSEZqf19i95gdEUcDg5Pa4OjbM1Qu69WczjuaWIL1XPMLYC0rUdogiIFavZdoSzQ6vl/kOUMa92qoAlLfCz3QlQbbO95zIRdTmsefTzzgkGXtPXcJ/T15iLygRtT5JAhy26vGrtur31lqv9upwa631va1Wm/N7W+tM3KpLo20gvOpbEG4ZZIl8jT2fAUojCki7OhpXx4bjs4Nm5Fms2PqzGcfySjGyZxx7QYnoyogiIBqVJaaulORwD6+tEWQdVdWPfG2FHlmNtoHwWh1WNfpar818L2q5NixRK2DPpx9rqBf0xp6x6BHPXlAiamPqBtmGAm1zw603emQBJXh6Glib857/Pac2gj2fbUBDvaBbfjLjaC57QYmojRE1VzYWtjbnigSNhld7zefmvgeq15S1td6kLyfXUINWDLScBEZ+jD2fAcIhydhz6hJ2sxeUiMi3nI+JvWxI9TTQ2lr2ZK7mEjUNB1NRV/3e+Vlb63td9ff6mkBcb38dx9JSo9jz2cZoRAHXXx2NlNhwfHrAjPwSpRf0WG4p/oe9oERE3iMI1Qv/61vvnLJcM9SgReG1kQDsHHIgOaofVVvZejU7ido64bSxoNpYsL1M0BU1HIYQBNjzGYCcvaD/PXEJkiwjRK/BjT3i0D0+nL2gRETBTHI03RsrVdVqswOSvZHP1ftJ1e2+iAuC2Mxg28IeXP5vpFex57MNc/aCOtcFzS+x4pOfLuBYXjj+p2ccQvX8sxIRBSVRo2ytsZpBbbV7ap1h1BVU7TXhtt7nxoJunWArOaqvI3lnXK2Ts9e2djh1BlNRWyu41v2sde+5bfA7HYclNBNTSgCLizBi4uDO2H1SGQt6LLcUvxZW4H96xqF7fITa5RERUVshCNW9iF6KDc71Z+sF2waCapPBtoHjXdepqh6a4IXhCE6ipiacunplmwq1Ovfe3Hrf1T1HYE8oa9E/RcuXL8fixYthNpvRr18/vPTSSxg8eHCD+65evRpvvvkmfv75ZwDAgAEDsGDBgkb3J89oRAFDUqKREheGTw/koqDEio9/vICj8SXsBSUiosDgXH8WrdxjC1T32lbVCqRV7kG1we8a2rf2d7WPqf7sHJbgHG/rfJqYNziHJzTYg1snuIbFAp2u814tLeBxMlm/fj0yMjKwcuVKpKWlYenSpRg1ahSOHDmCuLi4evtv374dEydOxNChQ2E0GrFo0SLcfPPNOHDgADp16tQqPwQpvaB3DO6M/568iD0nC9kLSkREBNRan1XnvWs4hyU0GFTrhto6wbVuiL3cZ2fAdQ5PgA1A2eVri+7qd+HT4wlHaWlpGDRoEJYtWwYAkCQJiYmJmDFjBmbOnNnk8Q6HA1FRUVi2bBkmTZrUrGtywpFn8iyV+PSg0gsKAN3jI3Bjz1j2ghIREQUqt4DbVFCtNVQhJArocI1PSvTKhCObzYZ9+/YhMzPT1SaKItLT07Fr165mnaO8vBx2ux3t27dvdB+r1Qqrtaa72mKxeFJm0IuLdO8FPZpbgl8Ly/E/PePQjb2gREREgaf2uNvWeBiDijyaklVQUACHw4H4+Hi39vj4eJjN5mad4x//+Ac6duyI9PT0RvfJysqCyWRybYmJiZ6USVDGgg5NicHEwYmIiTCg3ObA5h8v4OMfL6Dc5qVHzxERERE1wafrASxcuBDr1q3Dxo0bYTQ2Pqg4MzMTxcXFru3s2bM+rLJtcfaCpiW3hygIOJpbgrd2ncax3BK1SyMiIqIg5NFt95iYGGg0GuTm5rq15+bmokOHDpc99vnnn8fChQvx+eef49prr73svgaDAQaDwZPS6DI0ooChXWOQEheOzw6YUVBqw+YfL6BHh1Lc2CMOIfrAXrKBiIiIAodHPZ96vR4DBgxAdna2q02SJGRnZ2PIkCGNHvfcc8/h6aefxtatWzFw4MCWV0tXJD5SWRfU2Qt6xFyCN3edwvE89oISERGRb3h82z0jIwOrV6/GG2+8gUOHDuGBBx5AWVkZpk6dCgCYNGmS24SkRYsWYfbs2XjttdeQlJQEs9kMs9mM0tLS1vspqNm0GhFDu8bg9sGJiAnXo9zmwEc/XMAnP11Ahc2hdnlERETUxnm89s6ECROQn5+POXPmwGw2o3///ti6datrEtKZM2cg1nq01IoVK2Cz2fDnP//Z7Txz587FvHnzrqx6ajFnL+h/T17C3lOFOGIuwdlL5RjZKw5d4zgjnoiIiLzD43U+1cB1Pr0r11KJTw+YcbFUeZZuhFELU4gOUaF6RIXpYArRIypUB1OIDloNn1lLRERE9XllnU9qm+Jd64IqvaAllVUoqazCr4UVbvsJAhBh1CEqVId2oTq0C9UjKlSPdiE6RIbooBEFlX4CIiIiChQMnwRAGQs6rGsMBnSJQmG5DYVldhRV2FBUbkdhufJqq5JgqbDDUmHH6Yvux4uCgMgQrRJGXcFUh3YhekQYtRAZTImIiAgMn1SHUadBgikECSb3pyfIsoxym8MVRIvKlXBaWG5HcbkNdofsaq9LIwpoV33b3hlOna/hBi0EgcGUiIgoWDB8UrMIgoAwgxZhBi2uinL/TpZllFqrXOGzsNyGwnIbiiuUzw5JxsVSW/WY0jK3Y3UaAaZavaTtqm/pR4XqEarXMJgSERG1MQyfdMUEQUCEUYcIow6J7d2/kyQZJdYqFJUrvaRFrp5TG4orqmB3yCgosaKgxFrvvHqtWNNLGlJ9Kz9MCalcGJ+IiCgwMXySV4miAFOIcsu9S7T7dw5JhqXCjqIK57hS5xhTO0oqlTGmeRYr8iz1g6lRp3Gb+OQMqaYQHYw6BlMiIiJ/xfBJqtGIAqLC9IgK0yMZYW7fVTkkFFcoQbS4wjkBSukxLamsQqXdgQvFDlworqx33lC9Rgmi1YE0KlQHU/Vtfb2WS0URERGpieGT/JJWIyI63IDocEO97+wOyXXrvqjCjsIym2sCVJnVgXKbA+W2Cpwrqqh3bLhB6zbhydlr2o5rmBIREfkEwycFHJ1GRGyEAbER9YOptcqB4upb94W1xpcWlttRaXeg1FqFUmvDa5iGG7QI1WsRqtcgRK9BaPVm1Gnc23UaBlUiIqIWYvikNsWg1SAuUoO4SGO97yrtNUtF1V4yqrDcBluV5Fpcvzn0WtEVTkP0WoTqqoOqM7TqtK4AG6LTcJ1TIiKiagyfFDQut4Zphd0BS0UVym1VKLc5UGFXbt9XVH8utzlQWd3mkGTYqiTYqqQG1zWtSxBQ3XuqBFFnL6qzraanVWk3aEUuMUVERG0WwycFPUEQqoNf0/86yLIMa5WECpsD5Xb3cFphc443rXIF1Qq7A7IMVFR/3xyiILjd+g/RuYfTkDq9qzqNwLBKREQBg+GTyAOCIMCoU3oto5reHZIko7Kqfjh1vbc7UFndVm53wGqXIFUv2l9qbd4QAJ1GUG79uwVVZ3DVuvWuhnC8KhERqYzhk8iLRLH5vaqAssRUhb12UHWgwl7/1n+5zYFyaxWqJBl2hwx7hR2WiqaHAACAQSdW3/6vGa9q1Glg0InQa0TXq14rwqDVVL+KHA5AREStguGTyI9oNSIiNCIijLom95VlJXgqQwCq3HpXKxoZEiDJMqx2CVZ788ar1lU7iNYNp3qtM7xq3EKsQSfCoKkJt5x8RUQU3Bg+iQKUIAjQawXotSJMaF5YtVZJ9W/92xywVjlgrZ5EVfPqcE2sqpJkAHB9LrmCup0hVX+ZEFu7vXbYdbZpGGCJiAIWwydRkKg9XrV9mN6jY6scEmwOpce05tWBSnvddgdsjoZDrN3hHmBR/6mpzaYVhVo9q3V7Wut8biTccuwrEZE6GD6JqElajRLWQj3LrG6cS1Q5w6jVtdX0sLr1vjocdcJudWgFUCXJqLI6UAYHAM+HDwDK412dIVSvEaDViNBpROg0ArSi8qqrbtO63ivf6bXV+2hF6EShzj4ie2aJiC6D4ZOIfEIjCsqMe72mxeeQJNkVRq21wmlDPa0NfXa2AUoYLrc5ADRvCSxPiIIAnVaArjrEajU1YdYVdmuF19rfO4OsvvpVK4qu984AzIlfRBTIGD6JKGCIogCjqAwdQDPGuTbEOfbVGWKrJAn2Khl2SYLdIaHKIcPuUIYJOIcbuNokpc35vXN/5z6SrAwtUCZ2ybBCasWfvoYz0GpFZcxvwz211WG3dk+txj0Ma0Wll1YrCtBoql9FpV0UwJBLRF7B8ElEQaX22FfUfwrrFXFIzuBaP7xWSRJsVcpr3fDq2l+qmeBV+xzO907Ksa3fY1ubIKA6jIo1oVQj1ITV6sBb+3NNeK29fwPtolgn7Na0azUCNILAVRGI2jCGTyKiVqIRBWhcPbOty7m0Vu2e2prgWhNm7Y7Gw2vdY6skGQ5J2d9R/bnmer4JuY3R1AumAjSaRoJsI723GlGoFZBFt+NE0RlyAY3g3qYRBYiuV/YAE7U2hk8iogDgWloLInAFE78uR5ZlOCRnKFVeqxxSvTZnYK1y1N2/1r6ORtqrz+l+Ptlt2AKg9CI7JBk27/yozSYIcPXEasSaXlmNgDohVmggxKImxDYQbDVu50Sd8zMkU9vF8ElERACU8KLVCNC2fsdts0iNBVm3QNtwGLY76oTjesG4JjRL1W0OWbmmozp0O7faZBmokmWgTrs/coZZsTr4akTRLSSLghJaBWdbnfeigOpXJey63tf5rBGVf1Zqn69uCHa+FwXntWufr367Rqx1nupzMky3XQyfRETkF0RRgN411lOdBCzLMiRZ6XmVnKFUrh1YZUgSGmirCbGSBFRJUvXxjZ/L9X0j56odkh2SBIeEmvM0EIbrt6szZKK11Au61QFYEJSeZ1F0vq8faDXV34nVPddCrZDs3NcZcGt/FgUBAuA6j+D8LAju+9Q6l8fndp5LRLPO3Ra1KHwuX74cixcvhtlsRr9+/fDSSy9h8ODBje7//vvvY/bs2Th16hS6deuGRYsW4fe//32LiyYiIvIGZ7Dx97VaWxKSqyTZdZzzGNn5XpbrnVNu5L3zeNd7qeH3rnNWB2hZrgncUvV3zvcNkWQZkgMA/L/X2VsEARBQHWLF+iG2blCteXV+BySYQjC8e6zaP4obj8Pn+vXrkZGRgZUrVyItLQ1Lly7FqFGjcOTIEcTFxdXb/9tvv8XEiRORlZWFW265BWvXrsW4ceOwf/9+9O3bt1V+CCIiomASKCG5OWRnuJUbCbrNCL0OqZHw7DpnzTEy4Pq+oVdJrgn3rlfU/qwE+5rz1NQm1zqm0XOj+rNUs1/jv5uaa7d06Ic3JkBeKUGWL/NTNyAtLQ2DBg3CsmXLAACSJCExMREzZszAzJkz6+0/YcIElJWVYfPmza6266+/Hv3798fKlSubdU2LxQKTyYTi4mJERkZ6Ui4RERGRX6sdWGtCrHvYRp3PrhBbK1gDtT5XnydUr0HHdiE++Tmam9c86vm02WzYt28fMjMzXW2iKCI9PR27du1q8Jhdu3YhIyPDrW3UqFHYtGmTJ5cmIiIiapNcPdkI/J7s5vAofBYUFMDhcCA+Pt6tPT4+HocPH27wGLPZ3OD+ZrO50etYrVZYrVbXZ4vF4kmZREREROSnRLULaEhWVhZMJpNrS0xMVLskIiIiImoFHoXPmJgYaDQa5ObmurXn5uaiQ4cODR7ToUMHj/YHgMzMTBQXF7u2s2fPelImEREREfkpj8KnXq/HgAEDkJ2d7WqTJAnZ2dkYMmRIg8cMGTLEbX8A2LZtW6P7A4DBYEBkZKTbRkRERESBz+OlljIyMjB58mQMHDgQgwcPxtKlS1FWVoapU6cCACZNmoROnTohKysLAPDII4/ghhtuwJIlSzBmzBisW7cOe/fuxapVq1r3JyEiIiIiv+dx+JwwYQLy8/MxZ84cmM1m9O/fH1u3bnVNKjpz5gxEsaZDdejQoVi7di1mzZqFJ598Et26dcOmTZu4xicRERFREPJ4nU81cJ1PIiIiIv/W3Lzml7PdiYiIiKhtYvgkIiIiIp9h+CQiIiIin/F4wpEanMNS+aQjIiIiIv/kzGlNTScKiPBZUlICAHzSEREREZGfKykpgclkavT7gJjtLkkSzp8/j4iICAiC4PXrWSwWJCYm4uzZs5xdHyT4Nw9O/LsHH/7Ngw//5r4jyzJKSkrQsWNHt2U36wqInk9RFHHVVVf5/Lp8ulLw4d88OPHvHnz4Nw8+/Jv7xuV6PJ044YiIiIiIfIbhk4iIiIh8huGzAQaDAXPnzoXBYFC7FPIR/s2DE//uwYd/8+DDv7n/CYgJR0RERETUNrDnk4iIiIh8huGTiIiIiHyG4ZOIiIiIfIbhk4iIiIh8huGzAcuXL0dSUhKMRiPS0tKwe/dutUsiL8nKysKgQYMQERGBuLg4jBs3DkeOHFG7LPKhhQsXQhAEPProo2qXQl507tw53HXXXYiOjkZISAiuueYa7N27V+2yyIscDgdmz56N5ORkhISEICUlBU8//XSTzx0n72P4rGP9+vXIyMjA3LlzsX//fvTr1w+jRo1CXl6e2qWRF3z11VeYPn06vvvuO2zbtg12ux0333wzysrK1C6NfGDPnj145ZVXcO2116pdCnlRYWEhhg0bBp1Ohy1btuDgwYNYsmQJoqKi1C6NvGjRokVYsWIFli1bhkOHDmHRokV47rnn8NJLL6ldWtDjUkt1pKWlYdCgQVi2bBkA5bnyiYmJmDFjBmbOnKlydeRt+fn5iIuLw1dffYXhw4erXQ55UWlpKa677jq8/PLLeOaZZ9C/f38sXbpU7bLIC2bOnImdO3fi66+/VrsU8qFbbrkF8fHxePXVV11tt956K0JCQvD222+rWBmx57MWm82Gffv2IT093dUmiiLS09Oxa9cuFSsjXykuLgYAtG/fXuVKyNumT5+OMWPGuP37Tm3Thx9+iIEDB+J///d/ERcXh9TUVKxevVrtssjLhg4diuzsbBw9ehQA8MMPP+Cbb77B6NGjVa6MtGoX4E8KCgrgcDgQHx/v1h4fH4/Dhw+rVBX5iiRJePTRRzFs2DD07dtX7XLIi9atW4f9+/djz549apdCPnDixAmsWLECGRkZePLJJ7Fnzx48/PDD0Ov1mDx5strlkZfMnDkTFosFPXv2hEajgcPhwLPPPos777xT7dKCHsMnUbXp06fj559/xjfffKN2KeRFZ8+exSOPPIJt27bBaDSqXQ75gCRJGDhwIBYsWAAASE1Nxc8//4yVK1cyfLZh7733Ht555x2sXbsWffr0QU5ODh599FF07NiRf3eVMXzWEhMTA41Gg9zcXLf23NxcdOjQQaWqyBceeughbN68GTt27MBVV12ldjnkRfv27UNeXh6uu+46V5vD4cCOHTuwbNkyWK1WaDQaFSuk1paQkIDevXu7tfXq1Qv//ve/VaqIfOGJJ57AzJkzcfvttwMArrnmGpw+fRpZWVkMnyrjmM9a9Ho9BgwYgOzsbFebJEnIzs7GkCFDVKyMvEWWZTz00EPYuHEjvvjiCyQnJ6tdEnnZyJEj8dNPPyEnJ8e1DRw4EHfeeSdycnIYPNugYcOG1VtC7ejRo+jSpYtKFZEvlJeXQxTdY45Go4EkSSpVRE7s+awjIyMDkydPxsCBAzF48GAsXboUZWVlmDp1qtqlkRdMnz4da9euxX/+8x9ERETAbDYDAEwmE0JCQlSujrwhIiKi3pjesLAwREdHc6xvG/XYY49h6NChWLBgAW677Tbs3r0bq1atwqpVq9Qujbxo7NixePbZZ9G5c2f06dMH33//Pf71r39h2rRpapcW9LjUUgOWLVuGxYsXw2w2o3///njxxReRlpamdlnkBYIgNNj++uuvY8qUKb4thlQzYsQILrXUxm3evBmZmZk4duwYkpOTkZGRgXvuuUftssiLSkpKMHv2bGzcuBF5eXno2LEjJk6ciDlz5kCv16tdXlBj+CQiIiIin+GYTyIiIiLyGYZPIiIiIvIZhk8iIiIi8hmGTyIiIiLyGYZPIiIiIvIZhk8iIiIi8hmGTyIiIiLyGYZPIiIiIvIZhk8iIiIi8hmGTyIiIiLyGYZPIiIiIvIZhk8iIiIi8pn/D8vGt6MKuh/gAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "# compare\n",
        "fig, ax = plt.subplots(figsize=(8,4))\n",
        "plt.plot(losses_batchnorm, label='Using batchnorm', alpha=0.5)\n",
        "plt.plot(losses_no_norm, label='No norm', alpha=0.5)\n",
        "plt.title(\"Training Losses\")\n",
        "plt.legend()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": true,
        "id": "lU3lquGeA9_U"
      },
      "source": [
        "## Test\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "oelaAafzA9_U"
      },
      "outputs": [],
      "source": [
        "def test(model, train):\n",
        "    # zero array (size 10)\n",
        "    class_correct = list(0. for i in range(10))\n",
        "    class_total = list(0. for i in range(10))\n",
        "    test_loss = 0.0\n",
        "\n",
        "    # train parameter -> set model to training or evaluation mode\n",
        "    if(train==True):\n",
        "        model.train()\n",
        "    if(train==False):\n",
        "        model.eval()\n",
        "\n",
        "    # loss\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "    for batch_idx, (data, target) in enumerate(test_loader):\n",
        "        batch_size = data.size(0)\n",
        "        # forward\n",
        "        output = model(data)\n",
        "\n",
        "        # loss\n",
        "        loss = criterion(output, target)\n",
        "\n",
        "        test_loss += loss.item()*batch_size\n",
        "        # output -> predict class(target) using max prob\n",
        "        _, pred = torch.max(output, 1)\n",
        "        # compare predictions to true label\n",
        "        correct = np.squeeze(pred.eq(target.data.view_as(pred)))\n",
        "\n",
        "        # calculate test accuracy\n",
        "        for i in range(batch_size):\n",
        "            label = target.data[i]\n",
        "            class_correct[label] += correct[i].item()\n",
        "            class_total[label] += 1\n",
        "\n",
        "    # average test loss\n",
        "    print('Test Loss: {:.6f}\\n'.format(test_loss/len(test_loader.dataset)))\n",
        "\n",
        "    for i in range(10):\n",
        "        if class_total[i] > 0:\n",
        "            print('Test Accuracy of %5s: %2d%% (%2d/%2d)' % (str(i),\n",
        "                                                             100 * class_correct[i] / class_total[i],\n",
        "                                                             np.sum(class_correct[i]),\n",
        "                                                             np.sum(class_total[i])))\n",
        "        else:\n",
        "            print('Test Accuracy of %5s: N/A (no training examples)' % (str(i)))\n",
        "\n",
        "    print('\\nTest Accuracy (Overall): %2d%% (%2d/%2d)' % (100. * np.sum(class_correct) / np.sum(class_total),\n",
        "                                                          np.sum(class_correct), np.sum(class_total)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VqAIRNFvA9_V"
      },
      "source": [
        "### Train mode VS Evaluation mode\n",
        "\n",
        "Setting a model to evaluation mode is important for models with batch normalization layers\n",
        "\n",
        "- Train mode: batch normalization layers will use **batch** statistics to calculate the batch norm\n",
        "- Evaluation mode: use the estimated **population** mean and variance from the entire training set\n",
        "  -> give us increased performance on this test data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sm-a8Z71A9_W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4730a731-e1ae-4eb9-b124-6bfff2c01b92"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test Loss: 0.082439\n",
            "\n",
            "Test Accuracy of     0: 98% (966/980)\n",
            "Test Accuracy of     1: 99% (1127/1135)\n",
            "Test Accuracy of     2: 96% (997/1032)\n",
            "Test Accuracy of     3: 97% (982/1010)\n",
            "Test Accuracy of     4: 97% (954/982)\n",
            "Test Accuracy of     5: 97% (869/892)\n",
            "Test Accuracy of     6: 97% (936/958)\n",
            "Test Accuracy of     7: 97% (998/1028)\n",
            "Test Accuracy of     8: 97% (945/974)\n",
            "Test Accuracy of     9: 96% (976/1009)\n",
            "\n",
            "Test Accuracy (Overall): 97% (9750/10000)\n"
          ]
        }
      ],
      "source": [
        "# test (batch norm) in train mode\n",
        "test(net_batchnorm, train=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "D9e6bCSMA9_W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc8c54a0-ffe8-4e92-fb07-eb66011ee6bb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test Loss: 0.069304\n",
            "\n",
            "Test Accuracy of     0: 98% (968/980)\n",
            "Test Accuracy of     1: 99% (1127/1135)\n",
            "Test Accuracy of     2: 97% (1006/1032)\n",
            "Test Accuracy of     3: 98% (991/1010)\n",
            "Test Accuracy of     4: 97% (954/982)\n",
            "Test Accuracy of     5: 97% (870/892)\n",
            "Test Accuracy of     6: 97% (932/958)\n",
            "Test Accuracy of     7: 96% (990/1028)\n",
            "Test Accuracy of     8: 96% (941/974)\n",
            "Test Accuracy of     9: 98% (989/1009)\n",
            "\n",
            "Test Accuracy (Overall): 97% (9768/10000)\n"
          ]
        }
      ],
      "source": [
        "# test (batch norm) in eval mode\n",
        "test(net_batchnorm, train=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BmixGCRCA9_W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a24b39c9-1083-4edf-8834-dcc3e58f6d45"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test Loss: 0.207962\n",
            "\n",
            "Test Accuracy of     0: 98% (966/980)\n",
            "Test Accuracy of     1: 98% (1114/1135)\n",
            "Test Accuracy of     2: 91% (944/1032)\n",
            "Test Accuracy of     3: 93% (940/1010)\n",
            "Test Accuracy of     4: 93% (916/982)\n",
            "Test Accuracy of     5: 92% (822/892)\n",
            "Test Accuracy of     6: 95% (913/958)\n",
            "Test Accuracy of     7: 92% (952/1028)\n",
            "Test Accuracy of     8: 91% (890/974)\n",
            "Test Accuracy of     9: 93% (941/1009)\n",
            "\n",
            "Test Accuracy (Overall): 93% (9398/10000)\n"
          ]
        }
      ],
      "source": [
        "# test (without batch norm) in eval mode\n",
        "test(net_no_norm, train=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i12u0fjeA9_X"
      },
      "source": [
        "- batch norm model's accuracy: **evaluation mode** should give a small improvement (better than train mode)\n",
        "\n",
        "- using batch norm layers shows a marked improvement in overall accuracy when compared with the no-normalization model"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python [default]",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.4"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}