pytorch_ipynb/mlp/mlp-dropout.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
    "- Author: Sebastian Raschka\n",
    "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sebastian Raschka \n",
      "\n",
      "CPython 3.6.8\n",
      "IPython 7.2.0\n",
      "\n",
      "torch 1.0.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -v -p torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Runs on CPU or GPU (if available)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Zoo -- Multilayer Perceptron with Dropout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "from torchvision import datasets\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import DataLoader\n",
    "import torch.nn.functional as F\n",
    "import torch\n",
    "\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Settings and Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image batch dimensions: torch.Size([64, 1, 28, 28])\n",
      "Image label dimensions: torch.Size([64])\n"
     ]
    }
   ],
   "source": [
    "##########################\n",
    "### SETTINGS\n",
    "##########################\n",
    "\n",
    "# Device\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Hyperparameters\n",
    "random_seed = 1\n",
    "learning_rate = 0.1\n",
    "num_epochs = 10\n",
    "batch_size = 64\n",
    "dropout_prob = 0.5\n",
    "\n",
    "# Architecture\n",
    "num_features = 784\n",
    "num_hidden_1 = 128\n",
    "num_hidden_2 = 256\n",
    "num_classes = 10\n",
    "\n",
    "\n",
    "##########################\n",
    "### MNIST DATASET\n",
    "##########################\n",
    "\n",
    "# Note transforms.ToTensor() scales input images\n",
    "# to 0-1 range\n",
    "train_dataset = datasets.MNIST(root='data', \n",
    "                               train=True, \n",
    "                               transform=transforms.ToTensor(),\n",
    "                               download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='data', \n",
    "                              train=False, \n",
    "                              transform=transforms.ToTensor())\n",
    "\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, \n",
    "                          batch_size=batch_size, \n",
    "                          shuffle=True)\n",
    "\n",
    "test_loader = DataLoader(dataset=test_dataset, \n",
    "                         batch_size=batch_size, \n",
    "                         shuffle=False)\n",
    "\n",
    "# Checking the dataset\n",
    "for images, labels in train_loader:  \n",
    "    print('Image batch dimensions:', images.shape)\n",
    "    print('Image label dimensions:', labels.shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "##########################\n",
    "### MODEL\n",
    "##########################\n",
    "\n",
    "class MultilayerPerceptron(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, num_features, num_classes):\n",
    "        super(MultilayerPerceptron, self).__init__()\n",
    "        \n",
    "        ### 1st hidden layer\n",
    "        self.linear_1 = torch.nn.Linear(num_features, num_hidden_1)\n",
    "        # The following to lones are not necessary, \n",
    "        # but used here to demonstrate how to access the weights\n",
    "        # and use a different weight initialization.\n",
    "        # By default, PyTorch uses Xavier/Glorot initialization, which\n",
    "        # should usually be preferred.\n",
    "        self.linear_1.weight.detach().normal_(0.0, 0.1)\n",
    "        self.linear_1.bias.detach().zero_()\n",
    "        \n",
    "        ### 2nd hidden layer\n",
    "        self.linear_2 = torch.nn.Linear(num_hidden_1, num_hidden_2)\n",
    "        self.linear_2.weight.detach().normal_(0.0, 0.1)\n",
    "        self.linear_2.bias.detach().zero_()\n",
    "        \n",
    "        ### Output layer\n",
    "        self.linear_out = torch.nn.Linear(num_hidden_2, num_classes)\n",
    "        self.linear_out.weight.detach().normal_(0.0, 0.1)\n",
    "        self.linear_out.bias.detach().zero_()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = self.linear_1(x)\n",
    "        out = F.relu(out)\n",
    "        out = F.dropout(out, p=dropout_prob, training=self.training)\n",
    "        \n",
    "        out = self.linear_2(out)\n",
    "        out = F.relu(out)\n",
    "        out = F.dropout(out, p=dropout_prob, training=self.training)\n",
    "        \n",
    "        logits = self.linear_out(out)\n",
    "        probas = F.softmax(logits, dim=1)\n",
    "        return logits, probas\n",
    "\n",
    "    \n",
    "torch.manual_seed(random_seed)\n",
    "model = MultilayerPerceptron(num_features=num_features,\n",
    "                             num_classes=num_classes)\n",
    "\n",
    "model = model.to(device)\n",
    "\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001/010 | Batch 000/938 | Cost: 3.1761\n",
      "Epoch: 001/010 | Batch 050/938 | Cost: 1.2749\n",
      "Epoch: 001/010 | Batch 100/938 | Cost: 0.8759\n",
      "Epoch: 001/010 | Batch 150/938 | Cost: 0.9843\n",
      "Epoch: 001/010 | Batch 200/938 | Cost: 0.8911\n",
      "Epoch: 001/010 | Batch 250/938 | Cost: 0.6245\n",
      "Epoch: 001/010 | Batch 300/938 | Cost: 0.7050\n",
      "Epoch: 001/010 | Batch 350/938 | Cost: 0.6426\n",
      "Epoch: 001/010 | Batch 400/938 | Cost: 0.4462\n",
      "Epoch: 001/010 | Batch 450/938 | Cost: 0.5854\n",
      "Epoch: 001/010 | Batch 500/938 | Cost: 0.5844\n",
      "Epoch: 001/010 | Batch 550/938 | Cost: 0.4228\n",
      "Epoch: 001/010 | Batch 600/938 | Cost: 0.4705\n",
      "Epoch: 001/010 | Batch 650/938 | Cost: 0.7149\n",
      "Epoch: 001/010 | Batch 700/938 | Cost: 0.4342\n",
      "Epoch: 001/010 | Batch 750/938 | Cost: 0.5987\n",
      "Epoch: 001/010 | Batch 800/938 | Cost: 0.2601\n",
      "Epoch: 001/010 | Batch 850/938 | Cost: 0.2195\n",
      "Epoch: 001/010 | Batch 900/938 | Cost: 0.4569\n",
      "Epoch: 001/010 training accuracy: 93.04%\n",
      "Time elapsed: 0.22 min\n",
      "Epoch: 002/010 | Batch 000/938 | Cost: 0.6818\n",
      "Epoch: 002/010 | Batch 050/938 | Cost: 0.4469\n",
      "Epoch: 002/010 | Batch 100/938 | Cost: 0.4394\n",
      "Epoch: 002/010 | Batch 150/938 | Cost: 0.4237\n",
      "Epoch: 002/010 | Batch 200/938 | Cost: 0.4906\n",
      "Epoch: 002/010 | Batch 250/938 | Cost: 0.3429\n",
      "Epoch: 002/010 | Batch 300/938 | Cost: 0.2792\n",
      "Epoch: 002/010 | Batch 350/938 | Cost: 0.3293\n",
      "Epoch: 002/010 | Batch 400/938 | Cost: 0.3887\n",
      "Epoch: 002/010 | Batch 450/938 | Cost: 0.3144\n",
      "Epoch: 002/010 | Batch 500/938 | Cost: 0.4899\n",
      "Epoch: 002/010 | Batch 550/938 | Cost: 0.4949\n",
      "Epoch: 002/010 | Batch 600/938 | Cost: 0.4052\n",
      "Epoch: 002/010 | Batch 650/938 | Cost: 0.4248\n",
      "Epoch: 002/010 | Batch 700/938 | Cost: 0.4013\n",
      "Epoch: 002/010 | Batch 750/938 | Cost: 0.3184\n",
      "Epoch: 002/010 | Batch 800/938 | Cost: 0.5368\n",
      "Epoch: 002/010 | Batch 850/938 | Cost: 0.2178\n",
      "Epoch: 002/010 | Batch 900/938 | Cost: 0.2532\n",
      "Epoch: 002/010 training accuracy: 94.53%\n",
      "Time elapsed: 0.44 min\n",
      "Epoch: 003/010 | Batch 000/938 | Cost: 0.2330\n",
      "Epoch: 003/010 | Batch 050/938 | Cost: 0.2030\n",
      "Epoch: 003/010 | Batch 100/938 | Cost: 0.3366\n",
      "Epoch: 003/010 | Batch 150/938 | Cost: 0.4300\n",
      "Epoch: 003/010 | Batch 200/938 | Cost: 0.3449\n",
      "Epoch: 003/010 | Batch 250/938 | Cost: 0.5312\n",
      "Epoch: 003/010 | Batch 300/938 | Cost: 0.2596\n",
      "Epoch: 003/010 | Batch 350/938 | Cost: 0.2119\n",
      "Epoch: 003/010 | Batch 400/938 | Cost: 0.1706\n",
      "Epoch: 003/010 | Batch 450/938 | Cost: 0.1963\n",
      "Epoch: 003/010 | Batch 500/938 | Cost: 0.1826\n",
      "Epoch: 003/010 | Batch 550/938 | Cost: 0.1639\n",
      "Epoch: 003/010 | Batch 600/938 | Cost: 0.3906\n",
      "Epoch: 003/010 | Batch 650/938 | Cost: 0.2251\n",
      "Epoch: 003/010 | Batch 700/938 | Cost: 0.5097\n",
      "Epoch: 003/010 | Batch 750/938 | Cost: 0.1816\n",
      "Epoch: 003/010 | Batch 800/938 | Cost: 0.2478\n",
      "Epoch: 003/010 | Batch 850/938 | Cost: 0.0872\n",
      "Epoch: 003/010 | Batch 900/938 | Cost: 0.2131\n",
      "Epoch: 003/010 training accuracy: 95.74%\n",
      "Time elapsed: 0.66 min\n",
      "Epoch: 004/010 | Batch 000/938 | Cost: 0.0537\n",
      "Epoch: 004/010 | Batch 050/938 | Cost: 0.2216\n",
      "Epoch: 004/010 | Batch 100/938 | Cost: 0.2560\n",
      "Epoch: 004/010 | Batch 150/938 | Cost: 0.3367\n",
      "Epoch: 004/010 | Batch 200/938 | Cost: 0.2161\n",
      "Epoch: 004/010 | Batch 250/938 | Cost: 0.3530\n",
      "Epoch: 004/010 | Batch 300/938 | Cost: 0.4150\n",
      "Epoch: 004/010 | Batch 350/938 | Cost: 0.1628\n",
      "Epoch: 004/010 | Batch 400/938 | Cost: 0.3844\n",
      "Epoch: 004/010 | Batch 450/938 | Cost: 0.3700\n",
      "Epoch: 004/010 | Batch 500/938 | Cost: 0.3258\n",
      "Epoch: 004/010 | Batch 550/938 | Cost: 0.1491\n",
      "Epoch: 004/010 | Batch 600/938 | Cost: 0.4124\n",
      "Epoch: 004/010 | Batch 650/938 | Cost: 0.1568\n",
      "Epoch: 004/010 | Batch 700/938 | Cost: 0.2867\n",
      "Epoch: 004/010 | Batch 750/938 | Cost: 0.3083\n",
      "Epoch: 004/010 | Batch 800/938 | Cost: 0.2953\n",
      "Epoch: 004/010 | Batch 850/938 | Cost: 0.2130\n",
      "Epoch: 004/010 | Batch 900/938 | Cost: 0.1325\n",
      "Epoch: 004/010 training accuracy: 95.93%\n",
      "Time elapsed: 0.88 min\n",
      "Epoch: 005/010 | Batch 000/938 | Cost: 0.1164\n",
      "Epoch: 005/010 | Batch 050/938 | Cost: 0.2033\n",
      "Epoch: 005/010 | Batch 100/938 | Cost: 0.4225\n",
      "Epoch: 005/010 | Batch 150/938 | Cost: 0.2332\n",
      "Epoch: 005/010 | Batch 200/938 | Cost: 0.1807\n",
      "Epoch: 005/010 | Batch 250/938 | Cost: 0.2724\n",
      "Epoch: 005/010 | Batch 300/938 | Cost: 0.2070\n",
      "Epoch: 005/010 | Batch 350/938 | Cost: 0.3846\n",
      "Epoch: 005/010 | Batch 400/938 | Cost: 0.1403\n",
      "Epoch: 005/010 | Batch 450/938 | Cost: 0.1435\n",
      "Epoch: 005/010 | Batch 500/938 | Cost: 0.1864\n",
      "Epoch: 005/010 | Batch 550/938 | Cost: 0.4659\n",
      "Epoch: 005/010 | Batch 600/938 | Cost: 0.2498\n",
      "Epoch: 005/010 | Batch 650/938 | Cost: 0.1097\n",
      "Epoch: 005/010 | Batch 700/938 | Cost: 0.1233\n",
      "Epoch: 005/010 | Batch 750/938 | Cost: 0.1797\n",
      "Epoch: 005/010 | Batch 800/938 | Cost: 0.2743\n",
      "Epoch: 005/010 | Batch 850/938 | Cost: 0.4755\n",
      "Epoch: 005/010 | Batch 900/938 | Cost: 0.1791\n",
      "Epoch: 005/010 training accuracy: 96.62%\n",
      "Time elapsed: 1.10 min\n",
      "Epoch: 006/010 | Batch 000/938 | Cost: 0.2512\n",
      "Epoch: 006/010 | Batch 050/938 | Cost: 0.2439\n",
      "Epoch: 006/010 | Batch 100/938 | Cost: 0.2688\n",
      "Epoch: 006/010 | Batch 150/938 | Cost: 0.2428\n",
      "Epoch: 006/010 | Batch 200/938 | Cost: 0.1508\n",
      "Epoch: 006/010 | Batch 250/938 | Cost: 0.2942\n",
      "Epoch: 006/010 | Batch 300/938 | Cost: 0.3477\n",
      "Epoch: 006/010 | Batch 350/938 | Cost: 0.2686\n",
      "Epoch: 006/010 | Batch 400/938 | Cost: 0.1796\n",
      "Epoch: 006/010 | Batch 450/938 | Cost: 0.3615\n",
      "Epoch: 006/010 | Batch 500/938 | Cost: 0.1728\n",
      "Epoch: 006/010 | Batch 550/938 | Cost: 0.2942\n",
      "Epoch: 006/010 | Batch 600/938 | Cost: 0.2126\n",
      "Epoch: 006/010 | Batch 650/938 | Cost: 0.1768\n",
      "Epoch: 006/010 | Batch 700/938 | Cost: 0.3725\n",
      "Epoch: 006/010 | Batch 750/938 | Cost: 0.4141\n",
      "Epoch: 006/010 | Batch 800/938 | Cost: 0.0981\n",
      "Epoch: 006/010 | Batch 850/938 | Cost: 0.2725\n",
      "Epoch: 006/010 | Batch 900/938 | Cost: 0.3742\n",
      "Epoch: 006/010 training accuracy: 96.80%\n",
      "Time elapsed: 1.33 min\n",
      "Epoch: 007/010 | Batch 000/938 | Cost: 0.0982\n",
      "Epoch: 007/010 | Batch 050/938 | Cost: 0.3788\n",
      "Epoch: 007/010 | Batch 100/938 | Cost: 0.2841\n",
      "Epoch: 007/010 | Batch 150/938 | Cost: 0.2822\n",
      "Epoch: 007/010 | Batch 200/938 | Cost: 0.2435\n",
      "Epoch: 007/010 | Batch 250/938 | Cost: 0.1331\n",
      "Epoch: 007/010 | Batch 300/938 | Cost: 0.3305\n",
      "Epoch: 007/010 | Batch 350/938 | Cost: 0.3543\n",
      "Epoch: 007/010 | Batch 400/938 | Cost: 0.1692\n",
      "Epoch: 007/010 | Batch 450/938 | Cost: 0.2723\n",
      "Epoch: 007/010 | Batch 500/938 | Cost: 0.2608\n",
      "Epoch: 007/010 | Batch 550/938 | Cost: 0.2191\n",
      "Epoch: 007/010 | Batch 600/938 | Cost: 0.3432\n",
      "Epoch: 007/010 | Batch 650/938 | Cost: 0.2180\n",
      "Epoch: 007/010 | Batch 700/938 | Cost: 0.2242\n",
      "Epoch: 007/010 | Batch 750/938 | Cost: 0.2166\n",
      "Epoch: 007/010 | Batch 800/938 | Cost: 0.1156\n",
      "Epoch: 007/010 | Batch 850/938 | Cost: 0.1677\n",
      "Epoch: 007/010 | Batch 900/938 | Cost: 0.2352\n",
      "Epoch: 007/010 training accuracy: 97.08%\n",
      "Time elapsed: 1.55 min\n",
      "Epoch: 008/010 | Batch 000/938 | Cost: 0.2279\n",
      "Epoch: 008/010 | Batch 050/938 | Cost: 0.1192\n",
      "Epoch: 008/010 | Batch 100/938 | Cost: 0.3367\n",
      "Epoch: 008/010 | Batch 150/938 | Cost: 0.2009\n",
      "Epoch: 008/010 | Batch 200/938 | Cost: 0.1724\n",
      "Epoch: 008/010 | Batch 250/938 | Cost: 0.3747\n",
      "Epoch: 008/010 | Batch 300/938 | Cost: 0.3699\n",
      "Epoch: 008/010 | Batch 350/938 | Cost: 0.2708\n",
      "Epoch: 008/010 | Batch 400/938 | Cost: 0.1173\n",
      "Epoch: 008/010 | Batch 450/938 | Cost: 0.3007\n",
      "Epoch: 008/010 | Batch 500/938 | Cost: 0.1174\n",
      "Epoch: 008/010 | Batch 550/938 | Cost: 0.1924\n",
      "Epoch: 008/010 | Batch 600/938 | Cost: 0.0708\n",
      "Epoch: 008/010 | Batch 650/938 | Cost: 0.0882\n",
      "Epoch: 008/010 | Batch 700/938 | Cost: 0.1822\n",
      "Epoch: 008/010 | Batch 750/938 | Cost: 0.1415\n",
      "Epoch: 008/010 | Batch 800/938 | Cost: 0.1324\n",
      "Epoch: 008/010 | Batch 850/938 | Cost: 0.1612\n",
      "Epoch: 008/010 | Batch 900/938 | Cost: 0.2157\n",
      "Epoch: 008/010 training accuracy: 97.30%\n",
      "Time elapsed: 1.77 min\n",
      "Epoch: 009/010 | Batch 000/938 | Cost: 0.2361\n",
      "Epoch: 009/010 | Batch 050/938 | Cost: 0.2223\n",
      "Epoch: 009/010 | Batch 100/938 | Cost: 0.2047\n",
      "Epoch: 009/010 | Batch 150/938 | Cost: 0.0970\n",
      "Epoch: 009/010 | Batch 200/938 | Cost: 0.2133\n",
      "Epoch: 009/010 | Batch 250/938 | Cost: 0.0939\n",
      "Epoch: 009/010 | Batch 300/938 | Cost: 0.1779\n",
      "Epoch: 009/010 | Batch 350/938 | Cost: 0.0470\n",
      "Epoch: 009/010 | Batch 400/938 | Cost: 0.4539\n",
      "Epoch: 009/010 | Batch 450/938 | Cost: 0.1450\n",
      "Epoch: 009/010 | Batch 500/938 | Cost: 0.1942\n",
      "Epoch: 009/010 | Batch 550/938 | Cost: 0.2646\n",
      "Epoch: 009/010 | Batch 600/938 | Cost: 0.3475\n",
      "Epoch: 009/010 | Batch 650/938 | Cost: 0.1753\n",
      "Epoch: 009/010 | Batch 700/938 | Cost: 0.3570\n",
      "Epoch: 009/010 | Batch 750/938 | Cost: 0.2693\n",
      "Epoch: 009/010 | Batch 800/938 | Cost: 0.1132\n",
      "Epoch: 009/010 | Batch 850/938 | Cost: 0.4668\n",
      "Epoch: 009/010 | Batch 900/938 | Cost: 0.1920\n",
      "Epoch: 009/010 training accuracy: 97.38%\n",
      "Time elapsed: 1.99 min\n",
      "Epoch: 010/010 | Batch 000/938 | Cost: 0.1652\n",
      "Epoch: 010/010 | Batch 050/938 | Cost: 0.2654\n",
      "Epoch: 010/010 | Batch 100/938 | Cost: 0.1164\n",
      "Epoch: 010/010 | Batch 150/938 | Cost: 0.1916\n",
      "Epoch: 010/010 | Batch 200/938 | Cost: 0.1833\n",
      "Epoch: 010/010 | Batch 250/938 | Cost: 0.1914\n",
      "Epoch: 010/010 | Batch 300/938 | Cost: 0.1332\n",
      "Epoch: 010/010 | Batch 350/938 | Cost: 0.1535\n",
      "Epoch: 010/010 | Batch 400/938 | Cost: 0.0945\n",
      "Epoch: 010/010 | Batch 450/938 | Cost: 0.1842\n",
      "Epoch: 010/010 | Batch 500/938 | Cost: 0.2954\n",
      "Epoch: 010/010 | Batch 550/938 | Cost: 0.0577\n",
      "Epoch: 010/010 | Batch 600/938 | Cost: 0.1223\n",
      "Epoch: 010/010 | Batch 650/938 | Cost: 0.2175\n",
      "Epoch: 010/010 | Batch 700/938 | Cost: 0.2758\n",
      "Epoch: 010/010 | Batch 750/938 | Cost: 0.0905\n",
      "Epoch: 010/010 | Batch 800/938 | Cost: 0.1565\n",
      "Epoch: 010/010 | Batch 850/938 | Cost: 0.2303\n",
      "Epoch: 010/010 | Batch 900/938 | Cost: 0.1794\n",
      "Epoch: 010/010 training accuracy: 97.52%\n",
      "Time elapsed: 2.20 min\n",
      "Total Training Time: 2.20 min\n"
     ]
    }
   ],
   "source": [
    "def compute_accuracy(net, data_loader):\n",
    "    net.eval()\n",
    "    correct_pred, num_examples = 0, 0\n",
    "    with torch.no_grad():\n",
    "        for features, targets in data_loader:\n",
    "            features = features.view(-1, 28*28).to(device)\n",
    "            targets = targets.to(device)\n",
    "            logits, probas = net(features)\n",
    "            _, predicted_labels = torch.max(probas, 1)\n",
    "            num_examples += targets.size(0)\n",
    "            correct_pred += (predicted_labels == targets).sum()\n",
    "        return correct_pred.float()/num_examples * 100\n",
    "    \n",
    "\n",
    "start_time = time.time()\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
    "        \n",
    "        features = features.view(-1, 28*28).to(device)\n",
    "        targets = targets.to(device)\n",
    "            \n",
    "        ### FORWARD AND BACK PROP\n",
    "        logits, probas = model(features)\n",
    "        cost = F.cross_entropy(logits, targets)\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        cost.backward()\n",
    "        \n",
    "        ### UPDATE MODEL PARAMETERS\n",
    "        optimizer.step()\n",
    "        \n",
    "        ### LOGGING\n",
    "        if not batch_idx % 50:\n",
    "            print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
    "                   %(epoch+1, num_epochs, batch_idx, \n",
    "                     len(train_loader), cost))\n",
    "\n",
    "\n",
    "    print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
    "          epoch+1, num_epochs, \n",
    "          compute_accuracy(model, train_loader)))\n",
    "\n",
    "    print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
    "    \n",
    "print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 96.71%\n"
     ]
    }
   ],
   "source": [
    "print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "numpy       1.15.4\n",
      "torch       1.0.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -iv"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}