Neural_Net_Scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Building a multi-layer perceptron with one hidden layer from scratch and test it on MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigma(x):\n",
    "    return torch.tanh(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dsigma(x):\n",
    "    return 1 - torch.pow(sigma(x), 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss(v, t):\n",
    "    return torch.pow(torch.norm(v-t), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dloss(v, t):\n",
    "    return 2*(v-t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* Using MNIST\n",
      "** Reduce the data-set (use --full for the full thing)\n",
      "** Use 1000 train and 1000 test samples\n",
      "torch.Size([1000, 784])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aims/.local/lib/python3.7/site-packages/torchvision/datasets/mnist.py:53: UserWarning: train_data has been renamed data\n",
      "  warnings.warn(\"train_data has been renamed data\")\n",
      "/home/aims/.local/lib/python3.7/site-packages/torchvision/datasets/mnist.py:43: UserWarning: train_labels has been renamed targets\n",
      "  warnings.warn(\"train_labels has been renamed targets\")\n",
      "/home/aims/.local/lib/python3.7/site-packages/torchvision/datasets/mnist.py:58: UserWarning: test_data has been renamed data\n",
      "  warnings.warn(\"test_data has been renamed data\")\n",
      "/home/aims/.local/lib/python3.7/site-packages/torchvision/datasets/mnist.py:48: UserWarning: test_labels has been renamed targets\n",
      "  warnings.warn(\"test_labels has been renamed targets\")\n"
     ]
    }
   ],
   "source": [
    "import dlc_practical_prologue as prologue\n",
    "\n",
    "train_input, train_target, test_input,test_target = prologue.load_data(one_hot_labels = True,\n",
    "                                                                      normalize = True)\n",
    "\n",
    "train_target = train_target * 0.9\n",
    "test_target  = test_target * 0.9\n",
    "print(train_input.shape)\n",
    "\n",
    "nb_hidden = 50\n",
    "epsilon = 1e-6\n",
    "nb_classes = train_target.size(1)\n",
    "n_train_samples = train_input.size(0)\n",
    "#n_test_samples = test_input.size(0)\n",
    "learning_rate = 0.01\n",
    "step = learning_rate/n_train_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1000, 784])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_input.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1000, 10])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_target.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1000, 784])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_input.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1000, 10])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_target.shape "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1 = torch.empty(nb_hidden, train_input.size(1)).normal_(0, epsilon)\n",
    "b1 = torch.empty(nb_hidden).normal_(0, epsilon)\n",
    "w2 = torch.empty(nb_classes, nb_hidden).normal_(0, epsilon)\n",
    "b2 = torch.empty(nb_classes).normal_(0, epsilon)\n",
    "\n",
    "\n",
    "dl_dw1 = torch.empty(w1.size())\n",
    "dl_db1 = torch.empty(b1.size())\n",
    "dl_dw2 = torch.empty(w2.size())\n",
    "dl_db2 = torch.empty(b2.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_pass(w1, b1, w2, b2, x):\n",
    "    x0 = x\n",
    "    s1 = torch.mv(w1, x0) + b1\n",
    "    x1 = sigma(s1)\n",
    "    s2 = torch.mv(w2,x1) + b2\n",
    "    x2 = sigma(s2)\n",
    "    \n",
    "    return x0, s1, x1, s2, x2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_pass(w1, b1, w2, b2,\n",
    "                  t,\n",
    "                  x, s1, x1, s2, x2,\n",
    "                 dl_dw1, dl_db1, dl_dw2, dl_db2):\n",
    "    \n",
    "    x0 = x\n",
    "    dl_dx2 = loss(x2, t)\n",
    "    dl_ds2 = dsigma(s2) * dl_dx2\n",
    "    dl_dx1 = torch.mv(w2.t(), dl_ds2)\n",
    "    dl_ds1 = dsigma(s1) * dl_dx1\n",
    "    \n",
    "    dl_dw2.add_(torch.mm(dl_ds2.view(-1,1), x1.view(1, -1)))\n",
    "    dl_db2.add_(dl_ds2)\n",
    "    dl_dw1.add_(torch.mm(dl_ds1.view(-1, 1), x0.view(1,-1)))\n",
    "    dl_db1.add_(dl_ds1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 acc_train_loss 1475.75 acc_train_error 90.30% test_error 91.30%\n",
      "1 acc_train_loss 1554.80 acc_train_error 90.70% test_error 88.80%\n",
      "2 acc_train_loss 1645.54 acc_train_error 90.70% test_error 89.00%\n",
      "3 acc_train_loss 1759.16 acc_train_error 90.50% test_error 89.20%\n",
      "4 acc_train_loss 1933.35 acc_train_error 90.70% test_error 89.30%\n",
      "5 acc_train_loss 2304.88 acc_train_error 90.70% test_error 89.30%\n",
      "6 acc_train_loss 3408.90 acc_train_error 90.70% test_error 89.30%\n",
      "7 acc_train_loss 7022.70 acc_train_error 90.70% test_error 90.40%\n",
      "8 acc_train_loss 11533.95 acc_train_error 89.80% test_error 90.50%\n",
      "9 acc_train_loss 12116.67 acc_train_error 89.90% test_error 90.20%\n",
      "10 acc_train_loss 12281.89 acc_train_error 89.40% test_error 90.50%\n",
      "11 acc_train_loss 12363.32 acc_train_error 89.90% test_error 90.10%\n",
      "12 acc_train_loss 12412.28 acc_train_error 90.00% test_error 90.60%\n",
      "13 acc_train_loss 12445.05 acc_train_error 89.50% test_error 90.20%\n",
      "14 acc_train_loss 12468.58 acc_train_error 89.60% test_error 90.40%\n",
      "15 acc_train_loss 12486.26 acc_train_error 89.50% test_error 90.60%\n",
      "16 acc_train_loss 12500.08 acc_train_error 89.90% test_error 90.50%\n",
      "17 acc_train_loss 12511.13 acc_train_error 89.90% test_error 90.70%\n",
      "18 acc_train_loss 12520.21 acc_train_error 90.00% test_error 90.30%\n",
      "19 acc_train_loss 12527.77 acc_train_error 90.10% test_error 90.50%\n",
      "20 acc_train_loss 12534.18 acc_train_error 90.20% test_error 90.40%\n",
      "21 acc_train_loss 12539.66 acc_train_error 89.80% test_error 90.30%\n",
      "22 acc_train_loss 12544.42 acc_train_error 89.80% test_error 90.60%\n",
      "23 acc_train_loss 12548.59 acc_train_error 90.10% test_error 90.10%\n",
      "24 acc_train_loss 12552.27 acc_train_error 89.80% test_error 90.20%\n",
      "25 acc_train_loss 12555.53 acc_train_error 89.90% test_error 90.60%\n",
      "26 acc_train_loss 12558.46 acc_train_error 89.70% test_error 91.00%\n",
      "27 acc_train_loss 12561.08 acc_train_error 90.00% test_error 90.40%\n",
      "28 acc_train_loss 12563.46 acc_train_error 89.70% test_error 90.60%\n",
      "29 acc_train_loss 12565.62 acc_train_error 89.90% test_error 90.40%\n",
      "30 acc_train_loss 12567.59 acc_train_error 90.10% test_error 90.50%\n",
      "31 acc_train_loss 12569.39 acc_train_error 89.90% test_error 90.50%\n",
      "32 acc_train_loss 12571.06 acc_train_error 89.90% test_error 90.50%\n",
      "33 acc_train_loss 12572.59 acc_train_error 89.60% test_error 90.70%\n",
      "34 acc_train_loss 12574.01 acc_train_error 89.90% test_error 90.50%\n",
      "35 acc_train_loss 12575.32 acc_train_error 90.00% test_error 90.50%\n",
      "36 acc_train_loss 12576.54 acc_train_error 89.70% test_error 90.70%\n",
      "37 acc_train_loss 12577.69 acc_train_error 90.20% test_error 90.60%\n",
      "38 acc_train_loss 12578.75 acc_train_error 90.20% test_error 90.70%\n",
      "39 acc_train_loss 12579.76 acc_train_error 90.00% test_error 90.70%\n",
      "40 acc_train_loss 12580.69 acc_train_error 90.10% test_error 90.50%\n",
      "41 acc_train_loss 12581.57 acc_train_error 90.00% test_error 90.50%\n",
      "42 acc_train_loss 12582.41 acc_train_error 90.00% test_error 90.50%\n",
      "43 acc_train_loss 12583.20 acc_train_error 89.90% test_error 90.60%\n",
      "44 acc_train_loss 12583.94 acc_train_error 90.00% test_error 90.50%\n",
      "45 acc_train_loss 12584.63 acc_train_error 89.90% test_error 90.40%\n",
      "46 acc_train_loss 12585.30 acc_train_error 89.90% test_error 90.80%\n",
      "47 acc_train_loss 12585.94 acc_train_error 90.00% test_error 90.50%\n",
      "48 acc_train_loss 12586.54 acc_train_error 89.90% test_error 90.50%\n",
      "49 acc_train_loss 12587.12 acc_train_error 89.90% test_error 90.60%\n",
      "50 acc_train_loss 12587.66 acc_train_error 90.00% test_error 90.60%\n",
      "51 acc_train_loss 12588.18 acc_train_error 89.90% test_error 90.60%\n",
      "52 acc_train_loss 12588.67 acc_train_error 89.90% test_error 90.40%\n",
      "53 acc_train_loss 12589.15 acc_train_error 90.00% test_error 90.60%\n",
      "54 acc_train_loss 12589.62 acc_train_error 90.10% test_error 90.50%\n",
      "55 acc_train_loss 12590.04 acc_train_error 89.80% test_error 90.60%\n",
      "56 acc_train_loss 12590.46 acc_train_error 90.00% test_error 90.60%\n",
      "57 acc_train_loss 12590.86 acc_train_error 90.00% test_error 90.70%\n",
      "58 acc_train_loss 12591.24 acc_train_error 90.00% test_error 90.80%\n",
      "59 acc_train_loss 12591.61 acc_train_error 90.00% test_error 90.70%\n",
      "60 acc_train_loss 12591.98 acc_train_error 89.70% test_error 90.70%\n",
      "61 acc_train_loss 12592.31 acc_train_error 89.80% test_error 90.60%\n",
      "62 acc_train_loss 12592.64 acc_train_error 90.00% test_error 90.60%\n",
      "63 acc_train_loss 12592.97 acc_train_error 90.10% test_error 90.60%\n",
      "64 acc_train_loss 12593.26 acc_train_error 90.00% test_error 90.60%\n",
      "65 acc_train_loss 12593.58 acc_train_error 89.90% test_error 90.50%\n",
      "66 acc_train_loss 12593.84 acc_train_error 90.00% test_error 90.70%\n",
      "67 acc_train_loss 12594.12 acc_train_error 89.80% test_error 90.60%\n",
      "68 acc_train_loss 12594.40 acc_train_error 89.90% test_error 90.60%\n",
      "69 acc_train_loss 12594.65 acc_train_error 89.90% test_error 90.70%\n",
      "70 acc_train_loss 12594.90 acc_train_error 90.00% test_error 90.60%\n",
      "71 acc_train_loss 12595.15 acc_train_error 90.10% test_error 90.80%\n",
      "72 acc_train_loss 12595.37 acc_train_error 90.00% test_error 90.50%\n",
      "73 acc_train_loss 12595.60 acc_train_error 90.10% test_error 90.70%\n",
      "74 acc_train_loss 12595.83 acc_train_error 90.10% test_error 90.60%\n",
      "75 acc_train_loss 12596.04 acc_train_error 90.00% test_error 90.40%\n",
      "76 acc_train_loss 12596.25 acc_train_error 90.00% test_error 90.50%\n",
      "77 acc_train_loss 12596.45 acc_train_error 90.00% test_error 90.70%\n",
      "78 acc_train_loss 12596.64 acc_train_error 90.00% test_error 90.60%\n",
      "79 acc_train_loss 12596.83 acc_train_error 90.50% test_error 90.60%\n",
      "80 acc_train_loss 12597.02 acc_train_error 89.90% test_error 90.70%\n",
      "81 acc_train_loss 12597.20 acc_train_error 89.70% test_error 90.50%\n",
      "82 acc_train_loss 12597.38 acc_train_error 90.00% test_error 90.60%\n",
      "83 acc_train_loss 12597.55 acc_train_error 90.00% test_error 90.40%\n",
      "84 acc_train_loss 12597.71 acc_train_error 90.00% test_error 90.50%\n",
      "85 acc_train_loss 12597.87 acc_train_error 89.90% test_error 90.70%\n",
      "86 acc_train_loss 12598.03 acc_train_error 90.00% test_error 90.60%\n",
      "87 acc_train_loss 12598.19 acc_train_error 90.00% test_error 90.60%\n",
      "88 acc_train_loss 12598.34 acc_train_error 90.00% test_error 90.50%\n",
      "89 acc_train_loss 12598.48 acc_train_error 90.10% test_error 90.60%\n",
      "90 acc_train_loss 12598.62 acc_train_error 90.10% test_error 90.60%\n",
      "91 acc_train_loss 12598.76 acc_train_error 90.00% test_error 90.60%\n",
      "92 acc_train_loss 12598.89 acc_train_error 90.00% test_error 90.60%\n",
      "93 acc_train_loss 12599.03 acc_train_error 90.00% test_error 90.70%\n",
      "94 acc_train_loss 12599.17 acc_train_error 90.00% test_error 90.60%\n",
      "95 acc_train_loss 12599.29 acc_train_error 90.10% test_error 90.60%\n",
      "96 acc_train_loss 12599.40 acc_train_error 90.00% test_error 90.70%\n",
      "97 acc_train_loss 12599.52 acc_train_error 90.00% test_error 90.60%\n",
      "98 acc_train_loss 12599.65 acc_train_error 89.90% test_error 90.60%\n",
      "99 acc_train_loss 12599.77 acc_train_error 90.10% test_error 90.60%\n",
      "100 acc_train_loss 12599.87 acc_train_error 90.00% test_error 90.60%\n",
      "101 acc_train_loss 12599.98 acc_train_error 90.00% test_error 90.60%\n",
      "102 acc_train_loss 12600.09 acc_train_error 90.00% test_error 90.50%\n",
      "103 acc_train_loss 12600.20 acc_train_error 89.80% test_error 90.50%\n",
      "104 acc_train_loss 12600.31 acc_train_error 89.80% test_error 90.70%\n",
      "105 acc_train_loss 12600.41 acc_train_error 90.10% test_error 90.60%\n",
      "106 acc_train_loss 12600.50 acc_train_error 90.00% test_error 90.50%\n",
      "107 acc_train_loss 12600.59 acc_train_error 90.20% test_error 90.60%\n",
      "108 acc_train_loss 12600.69 acc_train_error 90.10% test_error 90.40%\n",
      "109 acc_train_loss 12600.79 acc_train_error 90.00% test_error 90.60%\n",
      "110 acc_train_loss 12600.89 acc_train_error 90.00% test_error 90.60%\n",
      "111 acc_train_loss 12600.97 acc_train_error 90.00% test_error 90.60%\n",
      "112 acc_train_loss 12601.07 acc_train_error 90.00% test_error 90.60%\n",
      "113 acc_train_loss 12601.15 acc_train_error 90.10% test_error 90.60%\n",
      "114 acc_train_loss 12601.23 acc_train_error 90.10% test_error 90.60%\n",
      "115 acc_train_loss 12601.31 acc_train_error 90.00% test_error 90.70%\n",
      "116 acc_train_loss 12601.39 acc_train_error 90.00% test_error 90.70%\n",
      "117 acc_train_loss 12601.47 acc_train_error 90.00% test_error 90.50%\n",
      "118 acc_train_loss 12601.55 acc_train_error 90.00% test_error 90.50%\n",
      "119 acc_train_loss 12601.63 acc_train_error 90.00% test_error 90.60%\n",
      "120 acc_train_loss 12601.70 acc_train_error 89.90% test_error 90.60%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "121 acc_train_loss 12601.78 acc_train_error 89.90% test_error 90.60%\n",
      "122 acc_train_loss 12601.85 acc_train_error 90.00% test_error 90.80%\n",
      "123 acc_train_loss 12601.92 acc_train_error 90.00% test_error 90.50%\n",
      "124 acc_train_loss 12602.00 acc_train_error 90.00% test_error 90.60%\n",
      "125 acc_train_loss 12602.08 acc_train_error 90.00% test_error 90.80%\n",
      "126 acc_train_loss 12602.15 acc_train_error 89.90% test_error 90.60%\n",
      "127 acc_train_loss 12602.22 acc_train_error 89.80% test_error 90.60%\n",
      "128 acc_train_loss 12602.29 acc_train_error 90.10% test_error 90.60%\n",
      "129 acc_train_loss 12602.35 acc_train_error 90.00% test_error 90.60%\n",
      "130 acc_train_loss 12602.42 acc_train_error 90.00% test_error 90.70%\n",
      "131 acc_train_loss 12602.47 acc_train_error 90.00% test_error 90.70%\n",
      "132 acc_train_loss 12602.53 acc_train_error 90.00% test_error 90.60%\n",
      "133 acc_train_loss 12602.58 acc_train_error 90.00% test_error 90.60%\n",
      "134 acc_train_loss 12602.64 acc_train_error 90.10% test_error 90.60%\n",
      "135 acc_train_loss 12602.69 acc_train_error 90.00% test_error 90.60%\n",
      "136 acc_train_loss 12602.75 acc_train_error 90.00% test_error 90.60%\n",
      "137 acc_train_loss 12602.80 acc_train_error 90.10% test_error 90.50%\n",
      "138 acc_train_loss 12602.85 acc_train_error 90.10% test_error 90.60%\n",
      "139 acc_train_loss 12602.91 acc_train_error 90.00% test_error 90.50%\n",
      "140 acc_train_loss 12602.97 acc_train_error 90.00% test_error 90.60%\n",
      "141 acc_train_loss 12603.03 acc_train_error 89.90% test_error 90.60%\n",
      "142 acc_train_loss 12603.09 acc_train_error 90.00% test_error 90.50%\n",
      "143 acc_train_loss 12603.14 acc_train_error 90.00% test_error 90.70%\n",
      "144 acc_train_loss 12603.19 acc_train_error 90.10% test_error 90.60%\n",
      "145 acc_train_loss 12603.25 acc_train_error 89.90% test_error 90.80%\n",
      "146 acc_train_loss 12603.30 acc_train_error 90.00% test_error 90.60%\n",
      "147 acc_train_loss 12603.35 acc_train_error 90.00% test_error 90.60%\n",
      "148 acc_train_loss 12603.40 acc_train_error 90.00% test_error 90.70%\n",
      "149 acc_train_loss 12603.45 acc_train_error 90.10% test_error 90.50%\n",
      "150 acc_train_loss 12603.51 acc_train_error 90.20% test_error 90.70%\n",
      "151 acc_train_loss 12603.55 acc_train_error 90.00% test_error 90.60%\n",
      "152 acc_train_loss 12603.59 acc_train_error 90.00% test_error 90.50%\n",
      "153 acc_train_loss 12603.64 acc_train_error 90.00% test_error 90.70%\n",
      "154 acc_train_loss 12603.68 acc_train_error 90.00% test_error 90.60%\n",
      "155 acc_train_loss 12603.73 acc_train_error 90.00% test_error 90.40%\n",
      "156 acc_train_loss 12603.76 acc_train_error 90.00% test_error 90.50%\n",
      "157 acc_train_loss 12603.80 acc_train_error 90.00% test_error 90.50%\n",
      "158 acc_train_loss 12603.84 acc_train_error 90.00% test_error 90.60%\n",
      "159 acc_train_loss 12603.88 acc_train_error 89.90% test_error 90.60%\n",
      "160 acc_train_loss 12603.92 acc_train_error 90.00% test_error 90.60%\n",
      "161 acc_train_loss 12603.96 acc_train_error 90.00% test_error 90.60%\n",
      "162 acc_train_loss 12604.00 acc_train_error 90.00% test_error 90.70%\n",
      "163 acc_train_loss 12604.03 acc_train_error 90.00% test_error 90.60%\n",
      "164 acc_train_loss 12604.06 acc_train_error 89.90% test_error 90.60%\n",
      "165 acc_train_loss 12604.10 acc_train_error 89.90% test_error 90.70%\n",
      "166 acc_train_loss 12604.13 acc_train_error 89.90% test_error 90.60%\n",
      "167 acc_train_loss 12604.17 acc_train_error 90.00% test_error 90.50%\n",
      "168 acc_train_loss 12604.21 acc_train_error 90.00% test_error 90.60%\n",
      "169 acc_train_loss 12604.25 acc_train_error 90.10% test_error 90.60%\n",
      "170 acc_train_loss 12604.29 acc_train_error 90.00% test_error 90.60%\n",
      "171 acc_train_loss 12604.33 acc_train_error 90.10% test_error 90.50%\n",
      "172 acc_train_loss 12604.37 acc_train_error 90.00% test_error 90.60%\n",
      "173 acc_train_loss 12604.40 acc_train_error 90.00% test_error 90.70%\n",
      "174 acc_train_loss 12604.44 acc_train_error 90.00% test_error 90.60%\n",
      "175 acc_train_loss 12604.47 acc_train_error 89.90% test_error 90.60%\n",
      "176 acc_train_loss 12604.50 acc_train_error 89.90% test_error 90.60%\n",
      "177 acc_train_loss 12604.54 acc_train_error 90.00% test_error 90.60%\n",
      "178 acc_train_loss 12604.57 acc_train_error 89.90% test_error 90.50%\n",
      "179 acc_train_loss 12604.61 acc_train_error 90.00% test_error 90.60%\n",
      "180 acc_train_loss 12604.64 acc_train_error 90.20% test_error 90.70%\n",
      "181 acc_train_loss 12604.68 acc_train_error 90.00% test_error 90.60%\n",
      "182 acc_train_loss 12604.72 acc_train_error 90.00% test_error 90.60%\n",
      "183 acc_train_loss 12604.75 acc_train_error 90.10% test_error 90.50%\n",
      "184 acc_train_loss 12604.78 acc_train_error 89.90% test_error 90.60%\n",
      "185 acc_train_loss 12604.81 acc_train_error 90.00% test_error 90.60%\n",
      "186 acc_train_loss 12604.83 acc_train_error 90.00% test_error 90.60%\n",
      "187 acc_train_loss 12604.87 acc_train_error 90.00% test_error 90.60%\n",
      "188 acc_train_loss 12604.90 acc_train_error 90.00% test_error 90.60%\n",
      "189 acc_train_loss 12604.93 acc_train_error 90.00% test_error 90.60%\n",
      "190 acc_train_loss 12604.96 acc_train_error 90.00% test_error 90.60%\n",
      "191 acc_train_loss 12605.00 acc_train_error 90.10% test_error 90.60%\n",
      "192 acc_train_loss 12605.02 acc_train_error 89.90% test_error 90.60%\n",
      "193 acc_train_loss 12605.04 acc_train_error 90.00% test_error 90.60%\n",
      "194 acc_train_loss 12605.07 acc_train_error 90.10% test_error 90.60%\n",
      "195 acc_train_loss 12605.10 acc_train_error 90.10% test_error 90.80%\n",
      "196 acc_train_loss 12605.12 acc_train_error 90.00% test_error 90.50%\n",
      "197 acc_train_loss 12605.15 acc_train_error 90.10% test_error 90.60%\n",
      "198 acc_train_loss 12605.17 acc_train_error 89.90% test_error 90.60%\n",
      "199 acc_train_loss 12605.19 acc_train_error 90.00% test_error 90.60%\n",
      "200 acc_train_loss 12605.21 acc_train_error 89.80% test_error 90.60%\n",
      "201 acc_train_loss 12605.24 acc_train_error 90.10% test_error 90.60%\n",
      "202 acc_train_loss 12605.26 acc_train_error 90.10% test_error 90.60%\n",
      "203 acc_train_loss 12605.28 acc_train_error 90.00% test_error 90.60%\n",
      "204 acc_train_loss 12605.31 acc_train_error 89.80% test_error 90.50%\n",
      "205 acc_train_loss 12605.33 acc_train_error 89.90% test_error 90.60%\n",
      "206 acc_train_loss 12605.35 acc_train_error 89.90% test_error 90.70%\n",
      "207 acc_train_loss 12605.37 acc_train_error 90.00% test_error 90.60%\n",
      "208 acc_train_loss 12605.39 acc_train_error 90.30% test_error 90.60%\n",
      "209 acc_train_loss 12605.41 acc_train_error 90.20% test_error 90.70%\n",
      "210 acc_train_loss 12605.43 acc_train_error 90.00% test_error 90.60%\n",
      "211 acc_train_loss 12605.46 acc_train_error 90.10% test_error 90.60%\n",
      "212 acc_train_loss 12605.47 acc_train_error 90.00% test_error 90.60%\n",
      "213 acc_train_loss 12605.50 acc_train_error 90.00% test_error 90.60%\n",
      "214 acc_train_loss 12605.52 acc_train_error 90.00% test_error 90.60%\n",
      "215 acc_train_loss 12605.54 acc_train_error 89.90% test_error 90.70%\n",
      "216 acc_train_loss 12605.56 acc_train_error 90.00% test_error 90.60%\n",
      "217 acc_train_loss 12605.58 acc_train_error 90.00% test_error 90.60%\n",
      "218 acc_train_loss 12605.60 acc_train_error 90.00% test_error 90.50%\n",
      "219 acc_train_loss 12605.62 acc_train_error 90.10% test_error 90.50%\n",
      "220 acc_train_loss 12605.64 acc_train_error 89.80% test_error 90.60%\n",
      "221 acc_train_loss 12605.67 acc_train_error 89.90% test_error 90.60%\n",
      "222 acc_train_loss 12605.69 acc_train_error 90.20% test_error 90.50%\n",
      "223 acc_train_loss 12605.71 acc_train_error 90.00% test_error 90.60%\n",
      "224 acc_train_loss 12605.73 acc_train_error 90.00% test_error 90.50%\n",
      "225 acc_train_loss 12605.75 acc_train_error 89.90% test_error 90.60%\n",
      "226 acc_train_loss 12605.77 acc_train_error 89.80% test_error 90.50%\n",
      "227 acc_train_loss 12605.79 acc_train_error 90.00% test_error 90.60%\n",
      "228 acc_train_loss 12605.80 acc_train_error 90.00% test_error 90.60%\n",
      "229 acc_train_loss 12605.83 acc_train_error 90.10% test_error 90.50%\n",
      "230 acc_train_loss 12605.85 acc_train_error 90.00% test_error 90.70%\n",
      "231 acc_train_loss 12605.87 acc_train_error 90.00% test_error 90.70%\n",
      "232 acc_train_loss 12605.89 acc_train_error 90.00% test_error 90.50%\n",
      "233 acc_train_loss 12605.91 acc_train_error 90.00% test_error 90.60%\n",
      "234 acc_train_loss 12605.93 acc_train_error 90.00% test_error 90.50%\n",
      "235 acc_train_loss 12605.96 acc_train_error 90.00% test_error 90.60%\n",
      "236 acc_train_loss 12605.98 acc_train_error 89.90% test_error 90.60%\n",
      "237 acc_train_loss 12606.00 acc_train_error 89.90% test_error 90.50%\n",
      "238 acc_train_loss 12606.03 acc_train_error 90.10% test_error 90.60%\n",
      "239 acc_train_loss 12606.04 acc_train_error 90.00% test_error 90.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "240 acc_train_loss 12606.06 acc_train_error 90.00% test_error 90.60%\n",
      "241 acc_train_loss 12606.09 acc_train_error 89.90% test_error 90.60%\n",
      "242 acc_train_loss 12606.10 acc_train_error 90.00% test_error 90.60%\n",
      "243 acc_train_loss 12606.12 acc_train_error 90.00% test_error 90.60%\n",
      "244 acc_train_loss 12606.14 acc_train_error 90.00% test_error 90.70%\n",
      "245 acc_train_loss 12606.17 acc_train_error 90.20% test_error 90.60%\n",
      "246 acc_train_loss 12606.18 acc_train_error 90.10% test_error 90.60%\n",
      "247 acc_train_loss 12606.20 acc_train_error 90.00% test_error 90.60%\n",
      "248 acc_train_loss 12606.22 acc_train_error 90.00% test_error 90.60%\n",
      "249 acc_train_loss 12606.23 acc_train_error 90.10% test_error 90.60%\n",
      "250 acc_train_loss 12606.24 acc_train_error 90.00% test_error 90.70%\n",
      "251 acc_train_loss 12606.26 acc_train_error 90.00% test_error 90.60%\n",
      "252 acc_train_loss 12606.28 acc_train_error 90.00% test_error 90.60%\n",
      "253 acc_train_loss 12606.29 acc_train_error 90.10% test_error 90.80%\n",
      "254 acc_train_loss 12606.31 acc_train_error 90.00% test_error 90.60%\n",
      "255 acc_train_loss 12606.33 acc_train_error 90.00% test_error 90.60%\n",
      "256 acc_train_loss 12606.34 acc_train_error 89.90% test_error 90.60%\n",
      "257 acc_train_loss 12606.36 acc_train_error 90.10% test_error 90.60%\n",
      "258 acc_train_loss 12606.37 acc_train_error 90.00% test_error 90.60%\n",
      "259 acc_train_loss 12606.38 acc_train_error 90.10% test_error 90.60%\n",
      "260 acc_train_loss 12606.39 acc_train_error 89.90% test_error 90.60%\n",
      "261 acc_train_loss 12606.41 acc_train_error 89.90% test_error 90.60%\n",
      "262 acc_train_loss 12606.42 acc_train_error 90.00% test_error 90.50%\n",
      "263 acc_train_loss 12606.43 acc_train_error 90.00% test_error 90.70%\n",
      "264 acc_train_loss 12606.45 acc_train_error 90.00% test_error 90.60%\n",
      "265 acc_train_loss 12606.46 acc_train_error 90.00% test_error 90.60%\n",
      "266 acc_train_loss 12606.48 acc_train_error 90.00% test_error 90.60%\n",
      "267 acc_train_loss 12606.49 acc_train_error 90.00% test_error 90.60%\n",
      "268 acc_train_loss 12606.50 acc_train_error 90.00% test_error 90.40%\n",
      "269 acc_train_loss 12606.51 acc_train_error 90.00% test_error 90.50%\n",
      "270 acc_train_loss 12606.53 acc_train_error 90.00% test_error 90.60%\n",
      "271 acc_train_loss 12606.54 acc_train_error 90.00% test_error 90.60%\n",
      "272 acc_train_loss 12606.55 acc_train_error 90.00% test_error 90.70%\n",
      "273 acc_train_loss 12606.56 acc_train_error 90.00% test_error 90.60%\n",
      "274 acc_train_loss 12606.57 acc_train_error 90.00% test_error 90.60%\n",
      "275 acc_train_loss 12606.59 acc_train_error 90.00% test_error 90.60%\n",
      "276 acc_train_loss 12606.60 acc_train_error 89.90% test_error 90.60%\n",
      "277 acc_train_loss 12606.61 acc_train_error 90.00% test_error 90.60%\n",
      "278 acc_train_loss 12606.63 acc_train_error 90.00% test_error 90.60%\n",
      "279 acc_train_loss 12606.63 acc_train_error 90.00% test_error 90.50%\n",
      "280 acc_train_loss 12606.64 acc_train_error 90.10% test_error 90.70%\n",
      "281 acc_train_loss 12606.65 acc_train_error 90.10% test_error 90.60%\n",
      "282 acc_train_loss 12606.67 acc_train_error 90.00% test_error 90.50%\n",
      "283 acc_train_loss 12606.68 acc_train_error 90.00% test_error 90.60%\n",
      "284 acc_train_loss 12606.69 acc_train_error 90.10% test_error 90.60%\n",
      "285 acc_train_loss 12606.70 acc_train_error 90.00% test_error 90.50%\n",
      "286 acc_train_loss 12606.70 acc_train_error 90.00% test_error 90.50%\n",
      "287 acc_train_loss 12606.71 acc_train_error 90.00% test_error 90.60%\n",
      "288 acc_train_loss 12606.72 acc_train_error 90.00% test_error 90.70%\n",
      "289 acc_train_loss 12606.74 acc_train_error 90.00% test_error 90.60%\n",
      "290 acc_train_loss 12606.75 acc_train_error 90.00% test_error 90.60%\n",
      "291 acc_train_loss 12606.75 acc_train_error 89.80% test_error 90.60%\n",
      "292 acc_train_loss 12606.76 acc_train_error 89.90% test_error 90.60%\n",
      "293 acc_train_loss 12606.77 acc_train_error 90.10% test_error 90.60%\n",
      "294 acc_train_loss 12606.78 acc_train_error 90.00% test_error 90.60%\n",
      "295 acc_train_loss 12606.79 acc_train_error 90.00% test_error 90.60%\n",
      "296 acc_train_loss 12606.80 acc_train_error 90.00% test_error 90.60%\n",
      "297 acc_train_loss 12606.81 acc_train_error 90.00% test_error 90.60%\n",
      "298 acc_train_loss 12606.82 acc_train_error 90.00% test_error 90.60%\n",
      "299 acc_train_loss 12606.83 acc_train_error 90.00% test_error 90.60%\n",
      "300 acc_train_loss 12606.84 acc_train_error 90.10% test_error 90.60%\n",
      "301 acc_train_loss 12606.85 acc_train_error 90.00% test_error 90.60%\n",
      "302 acc_train_loss 12606.86 acc_train_error 90.10% test_error 90.70%\n",
      "303 acc_train_loss 12606.87 acc_train_error 90.10% test_error 90.60%\n",
      "304 acc_train_loss 12606.88 acc_train_error 90.00% test_error 90.60%\n",
      "305 acc_train_loss 12606.89 acc_train_error 90.10% test_error 90.70%\n",
      "306 acc_train_loss 12606.90 acc_train_error 89.90% test_error 90.60%\n",
      "307 acc_train_loss 12606.91 acc_train_error 90.10% test_error 90.60%\n",
      "308 acc_train_loss 12606.92 acc_train_error 90.00% test_error 90.60%\n",
      "309 acc_train_loss 12606.92 acc_train_error 90.00% test_error 90.60%\n",
      "310 acc_train_loss 12606.93 acc_train_error 90.10% test_error 90.50%\n",
      "311 acc_train_loss 12606.94 acc_train_error 90.10% test_error 90.70%\n",
      "312 acc_train_loss 12606.95 acc_train_error 90.10% test_error 90.70%\n",
      "313 acc_train_loss 12606.96 acc_train_error 89.90% test_error 90.60%\n",
      "314 acc_train_loss 12606.96 acc_train_error 90.00% test_error 90.60%\n",
      "315 acc_train_loss 12606.98 acc_train_error 90.00% test_error 90.50%\n",
      "316 acc_train_loss 12606.99 acc_train_error 90.00% test_error 90.60%\n",
      "317 acc_train_loss 12607.00 acc_train_error 89.90% test_error 90.50%\n",
      "318 acc_train_loss 12607.01 acc_train_error 90.00% test_error 90.60%\n",
      "319 acc_train_loss 12607.02 acc_train_error 89.90% test_error 90.70%\n",
      "320 acc_train_loss 12607.03 acc_train_error 90.00% test_error 90.60%\n",
      "321 acc_train_loss 12607.04 acc_train_error 90.00% test_error 90.60%\n",
      "322 acc_train_loss 12607.05 acc_train_error 90.00% test_error 90.60%\n",
      "323 acc_train_loss 12607.07 acc_train_error 90.10% test_error 90.60%\n",
      "324 acc_train_loss 12607.08 acc_train_error 90.00% test_error 90.60%\n",
      "325 acc_train_loss 12607.09 acc_train_error 90.00% test_error 90.70%\n",
      "326 acc_train_loss 12607.11 acc_train_error 90.00% test_error 90.60%\n",
      "327 acc_train_loss 12607.11 acc_train_error 90.00% test_error 90.60%\n",
      "328 acc_train_loss 12607.13 acc_train_error 90.00% test_error 90.60%\n",
      "329 acc_train_loss 12607.14 acc_train_error 90.00% test_error 90.60%\n",
      "330 acc_train_loss 12607.15 acc_train_error 90.00% test_error 90.60%\n",
      "331 acc_train_loss 12607.17 acc_train_error 90.10% test_error 90.60%\n",
      "332 acc_train_loss 12607.17 acc_train_error 90.00% test_error 90.50%\n",
      "333 acc_train_loss 12607.19 acc_train_error 90.10% test_error 90.60%\n",
      "334 acc_train_loss 12607.20 acc_train_error 90.00% test_error 90.50%\n",
      "335 acc_train_loss 12607.20 acc_train_error 90.00% test_error 90.60%\n",
      "336 acc_train_loss 12607.21 acc_train_error 89.90% test_error 90.60%\n",
      "337 acc_train_loss 12607.22 acc_train_error 90.00% test_error 90.60%\n",
      "338 acc_train_loss 12607.23 acc_train_error 89.90% test_error 90.70%\n",
      "339 acc_train_loss 12607.25 acc_train_error 89.90% test_error 90.70%\n",
      "340 acc_train_loss 12607.26 acc_train_error 90.10% test_error 90.60%\n",
      "341 acc_train_loss 12607.27 acc_train_error 90.00% test_error 90.60%\n",
      "342 acc_train_loss 12607.28 acc_train_error 90.00% test_error 90.80%\n",
      "343 acc_train_loss 12607.30 acc_train_error 90.00% test_error 90.60%\n",
      "344 acc_train_loss 12607.31 acc_train_error 90.00% test_error 90.60%\n",
      "345 acc_train_loss 12607.32 acc_train_error 90.00% test_error 90.60%\n",
      "346 acc_train_loss 12607.33 acc_train_error 90.00% test_error 90.60%\n",
      "347 acc_train_loss 12607.33 acc_train_error 89.90% test_error 90.60%\n",
      "348 acc_train_loss 12607.34 acc_train_error 90.00% test_error 90.60%\n",
      "349 acc_train_loss 12607.35 acc_train_error 90.00% test_error 90.60%\n",
      "350 acc_train_loss 12607.36 acc_train_error 90.00% test_error 90.60%\n",
      "351 acc_train_loss 12607.37 acc_train_error 90.00% test_error 90.60%\n",
      "352 acc_train_loss 12607.38 acc_train_error 90.00% test_error 90.60%\n",
      "353 acc_train_loss 12607.39 acc_train_error 90.00% test_error 90.60%\n",
      "354 acc_train_loss 12607.40 acc_train_error 90.10% test_error 90.60%\n",
      "355 acc_train_loss 12607.41 acc_train_error 90.00% test_error 90.70%\n",
      "356 acc_train_loss 12607.41 acc_train_error 90.00% test_error 90.60%\n",
      "357 acc_train_loss 12607.42 acc_train_error 90.00% test_error 90.60%\n",
      "358 acc_train_loss 12607.43 acc_train_error 90.00% test_error 90.60%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "359 acc_train_loss 12607.44 acc_train_error 90.00% test_error 90.60%\n",
      "360 acc_train_loss 12607.44 acc_train_error 90.00% test_error 90.60%\n",
      "361 acc_train_loss 12607.45 acc_train_error 90.00% test_error 90.60%\n",
      "362 acc_train_loss 12607.46 acc_train_error 89.80% test_error 90.60%\n",
      "363 acc_train_loss 12607.46 acc_train_error 90.00% test_error 90.60%\n",
      "364 acc_train_loss 12607.47 acc_train_error 89.90% test_error 90.50%\n",
      "365 acc_train_loss 12607.48 acc_train_error 90.00% test_error 90.50%\n",
      "366 acc_train_loss 12607.49 acc_train_error 90.00% test_error 90.60%\n",
      "367 acc_train_loss 12607.50 acc_train_error 90.10% test_error 90.50%\n",
      "368 acc_train_loss 12607.51 acc_train_error 89.90% test_error 90.60%\n",
      "369 acc_train_loss 12607.52 acc_train_error 90.00% test_error 90.60%\n",
      "370 acc_train_loss 12607.53 acc_train_error 89.90% test_error 90.60%\n",
      "371 acc_train_loss 12607.53 acc_train_error 90.00% test_error 90.50%\n",
      "372 acc_train_loss 12607.53 acc_train_error 89.90% test_error 90.70%\n",
      "373 acc_train_loss 12607.54 acc_train_error 90.00% test_error 90.70%\n",
      "374 acc_train_loss 12607.55 acc_train_error 90.00% test_error 90.60%\n",
      "375 acc_train_loss 12607.55 acc_train_error 89.90% test_error 90.60%\n",
      "376 acc_train_loss 12607.56 acc_train_error 90.10% test_error 90.60%\n",
      "377 acc_train_loss 12607.56 acc_train_error 90.00% test_error 90.80%\n",
      "378 acc_train_loss 12607.57 acc_train_error 90.10% test_error 90.60%\n",
      "379 acc_train_loss 12607.58 acc_train_error 90.00% test_error 90.60%\n",
      "380 acc_train_loss 12607.58 acc_train_error 90.00% test_error 90.70%\n",
      "381 acc_train_loss 12607.59 acc_train_error 89.90% test_error 90.50%\n",
      "382 acc_train_loss 12607.60 acc_train_error 90.00% test_error 90.70%\n",
      "383 acc_train_loss 12607.61 acc_train_error 90.00% test_error 90.60%\n",
      "384 acc_train_loss 12607.62 acc_train_error 90.00% test_error 90.60%\n",
      "385 acc_train_loss 12607.63 acc_train_error 90.00% test_error 90.60%\n",
      "386 acc_train_loss 12607.63 acc_train_error 90.00% test_error 90.60%\n",
      "387 acc_train_loss 12607.63 acc_train_error 90.00% test_error 90.60%\n",
      "388 acc_train_loss 12607.64 acc_train_error 90.00% test_error 90.70%\n",
      "389 acc_train_loss 12607.65 acc_train_error 90.00% test_error 90.60%\n",
      "390 acc_train_loss 12607.65 acc_train_error 89.90% test_error 90.60%\n",
      "391 acc_train_loss 12607.66 acc_train_error 90.00% test_error 90.60%\n",
      "392 acc_train_loss 12607.67 acc_train_error 90.00% test_error 90.60%\n",
      "393 acc_train_loss 12607.68 acc_train_error 90.00% test_error 90.60%\n",
      "394 acc_train_loss 12607.68 acc_train_error 90.00% test_error 90.60%\n",
      "395 acc_train_loss 12607.69 acc_train_error 90.00% test_error 90.70%\n",
      "396 acc_train_loss 12607.70 acc_train_error 90.00% test_error 90.70%\n",
      "397 acc_train_loss 12607.71 acc_train_error 89.90% test_error 90.60%\n",
      "398 acc_train_loss 12607.71 acc_train_error 90.00% test_error 90.70%\n",
      "399 acc_train_loss 12607.72 acc_train_error 90.00% test_error 90.60%\n",
      "400 acc_train_loss 12607.72 acc_train_error 89.90% test_error 90.60%\n",
      "401 acc_train_loss 12607.73 acc_train_error 90.00% test_error 90.70%\n",
      "402 acc_train_loss 12607.74 acc_train_error 90.10% test_error 90.60%\n",
      "403 acc_train_loss 12607.74 acc_train_error 90.00% test_error 90.70%\n",
      "404 acc_train_loss 12607.75 acc_train_error 90.00% test_error 90.60%\n",
      "405 acc_train_loss 12607.76 acc_train_error 89.90% test_error 90.60%\n",
      "406 acc_train_loss 12607.76 acc_train_error 90.00% test_error 90.50%\n",
      "407 acc_train_loss 12607.77 acc_train_error 90.00% test_error 90.70%\n",
      "408 acc_train_loss 12607.77 acc_train_error 89.90% test_error 90.60%\n",
      "409 acc_train_loss 12607.78 acc_train_error 90.00% test_error 90.60%\n",
      "410 acc_train_loss 12607.78 acc_train_error 90.00% test_error 90.60%\n",
      "411 acc_train_loss 12607.78 acc_train_error 90.00% test_error 90.60%\n",
      "412 acc_train_loss 12607.79 acc_train_error 89.90% test_error 90.60%\n",
      "413 acc_train_loss 12607.79 acc_train_error 90.00% test_error 90.60%\n",
      "414 acc_train_loss 12607.80 acc_train_error 90.10% test_error 90.60%\n",
      "415 acc_train_loss 12607.80 acc_train_error 90.00% test_error 90.60%\n",
      "416 acc_train_loss 12607.81 acc_train_error 90.00% test_error 90.60%\n",
      "417 acc_train_loss 12607.81 acc_train_error 89.90% test_error 90.70%\n",
      "418 acc_train_loss 12607.82 acc_train_error 90.00% test_error 90.70%\n",
      "419 acc_train_loss 12607.82 acc_train_error 90.00% test_error 90.60%\n",
      "420 acc_train_loss 12607.83 acc_train_error 90.00% test_error 90.60%\n",
      "421 acc_train_loss 12607.83 acc_train_error 89.90% test_error 90.50%\n",
      "422 acc_train_loss 12607.84 acc_train_error 90.00% test_error 90.60%\n",
      "423 acc_train_loss 12607.84 acc_train_error 89.90% test_error 90.70%\n",
      "424 acc_train_loss 12607.85 acc_train_error 89.90% test_error 90.70%\n",
      "425 acc_train_loss 12607.85 acc_train_error 90.00% test_error 90.60%\n",
      "426 acc_train_loss 12607.85 acc_train_error 90.00% test_error 90.70%\n",
      "427 acc_train_loss 12607.86 acc_train_error 90.00% test_error 90.60%\n",
      "428 acc_train_loss 12607.86 acc_train_error 90.00% test_error 90.60%\n",
      "429 acc_train_loss 12607.87 acc_train_error 89.90% test_error 90.60%\n",
      "430 acc_train_loss 12607.87 acc_train_error 90.10% test_error 90.60%\n",
      "431 acc_train_loss 12607.88 acc_train_error 90.00% test_error 90.60%\n",
      "432 acc_train_loss 12607.88 acc_train_error 90.00% test_error 90.50%\n",
      "433 acc_train_loss 12607.89 acc_train_error 90.00% test_error 90.60%\n",
      "434 acc_train_loss 12607.89 acc_train_error 90.00% test_error 90.60%\n",
      "435 acc_train_loss 12607.90 acc_train_error 90.00% test_error 90.70%\n",
      "436 acc_train_loss 12607.90 acc_train_error 90.00% test_error 90.70%\n",
      "437 acc_train_loss 12607.90 acc_train_error 90.00% test_error 90.60%\n",
      "438 acc_train_loss 12607.91 acc_train_error 90.00% test_error 90.60%\n",
      "439 acc_train_loss 12607.91 acc_train_error 90.00% test_error 90.60%\n",
      "440 acc_train_loss 12607.92 acc_train_error 90.00% test_error 90.60%\n",
      "441 acc_train_loss 12607.92 acc_train_error 90.00% test_error 90.70%\n",
      "442 acc_train_loss 12607.93 acc_train_error 89.90% test_error 90.60%\n",
      "443 acc_train_loss 12607.93 acc_train_error 89.90% test_error 90.60%\n",
      "444 acc_train_loss 12607.93 acc_train_error 90.00% test_error 90.60%\n",
      "445 acc_train_loss 12607.94 acc_train_error 90.00% test_error 90.60%\n",
      "446 acc_train_loss 12607.94 acc_train_error 90.10% test_error 90.60%\n",
      "447 acc_train_loss 12607.95 acc_train_error 90.00% test_error 90.60%\n",
      "448 acc_train_loss 12607.95 acc_train_error 90.00% test_error 90.60%\n",
      "449 acc_train_loss 12607.96 acc_train_error 90.00% test_error 90.60%\n",
      "450 acc_train_loss 12607.96 acc_train_error 90.00% test_error 90.60%\n",
      "451 acc_train_loss 12607.96 acc_train_error 89.90% test_error 90.60%\n",
      "452 acc_train_loss 12607.96 acc_train_error 90.00% test_error 90.60%\n",
      "453 acc_train_loss 12607.97 acc_train_error 90.10% test_error 90.60%\n",
      "454 acc_train_loss 12607.97 acc_train_error 90.00% test_error 90.50%\n",
      "455 acc_train_loss 12607.98 acc_train_error 90.00% test_error 90.60%\n",
      "456 acc_train_loss 12607.98 acc_train_error 89.90% test_error 90.70%\n",
      "457 acc_train_loss 12607.99 acc_train_error 90.00% test_error 90.60%\n",
      "458 acc_train_loss 12607.99 acc_train_error 90.10% test_error 90.60%\n",
      "459 acc_train_loss 12607.99 acc_train_error 90.00% test_error 90.60%\n",
      "460 acc_train_loss 12607.99 acc_train_error 90.00% test_error 90.60%\n",
      "461 acc_train_loss 12608.00 acc_train_error 90.00% test_error 90.40%\n",
      "462 acc_train_loss 12608.00 acc_train_error 90.00% test_error 90.60%\n",
      "463 acc_train_loss 12608.00 acc_train_error 90.10% test_error 90.60%\n",
      "464 acc_train_loss 12608.01 acc_train_error 90.00% test_error 90.60%\n",
      "465 acc_train_loss 12608.01 acc_train_error 89.90% test_error 90.60%\n",
      "466 acc_train_loss 12608.02 acc_train_error 90.00% test_error 90.60%\n",
      "467 acc_train_loss 12608.02 acc_train_error 90.00% test_error 90.60%\n",
      "468 acc_train_loss 12608.02 acc_train_error 90.00% test_error 90.60%\n",
      "469 acc_train_loss 12608.02 acc_train_error 90.00% test_error 90.60%\n",
      "470 acc_train_loss 12608.03 acc_train_error 90.00% test_error 90.60%\n",
      "471 acc_train_loss 12608.03 acc_train_error 90.00% test_error 90.70%\n",
      "472 acc_train_loss 12608.03 acc_train_error 89.90% test_error 90.60%\n",
      "473 acc_train_loss 12608.04 acc_train_error 90.00% test_error 90.50%\n",
      "474 acc_train_loss 12608.04 acc_train_error 90.00% test_error 90.60%\n",
      "475 acc_train_loss 12608.04 acc_train_error 90.00% test_error 90.60%\n",
      "476 acc_train_loss 12608.04 acc_train_error 90.00% test_error 90.60%\n",
      "477 acc_train_loss 12608.05 acc_train_error 90.00% test_error 90.60%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "478 acc_train_loss 12608.05 acc_train_error 90.00% test_error 90.60%\n",
      "479 acc_train_loss 12608.05 acc_train_error 90.00% test_error 90.60%\n",
      "480 acc_train_loss 12608.06 acc_train_error 90.00% test_error 90.60%\n",
      "481 acc_train_loss 12608.06 acc_train_error 90.00% test_error 90.60%\n",
      "482 acc_train_loss 12608.06 acc_train_error 90.00% test_error 90.60%\n",
      "483 acc_train_loss 12608.07 acc_train_error 89.90% test_error 90.60%\n",
      "484 acc_train_loss 12608.07 acc_train_error 89.90% test_error 90.50%\n",
      "485 acc_train_loss 12608.08 acc_train_error 90.00% test_error 90.60%\n",
      "486 acc_train_loss 12608.08 acc_train_error 90.00% test_error 90.60%\n",
      "487 acc_train_loss 12608.08 acc_train_error 90.00% test_error 90.60%\n",
      "488 acc_train_loss 12608.09 acc_train_error 90.00% test_error 90.60%\n",
      "489 acc_train_loss 12608.09 acc_train_error 90.00% test_error 90.60%\n",
      "490 acc_train_loss 12608.10 acc_train_error 90.00% test_error 90.60%\n",
      "491 acc_train_loss 12608.10 acc_train_error 90.00% test_error 90.60%\n",
      "492 acc_train_loss 12608.10 acc_train_error 90.00% test_error 90.60%\n",
      "493 acc_train_loss 12608.11 acc_train_error 90.00% test_error 90.60%\n",
      "494 acc_train_loss 12608.11 acc_train_error 90.10% test_error 90.70%\n",
      "495 acc_train_loss 12608.12 acc_train_error 90.00% test_error 90.70%\n",
      "496 acc_train_loss 12608.12 acc_train_error 90.00% test_error 90.60%\n",
      "497 acc_train_loss 12608.13 acc_train_error 90.00% test_error 90.60%\n",
      "498 acc_train_loss 12608.13 acc_train_error 90.00% test_error 90.60%\n",
      "499 acc_train_loss 12608.13 acc_train_error 90.00% test_error 90.60%\n",
      "500 acc_train_loss 12608.13 acc_train_error 90.00% test_error 90.60%\n",
      "501 acc_train_loss 12608.14 acc_train_error 90.00% test_error 90.60%\n",
      "502 acc_train_loss 12608.14 acc_train_error 90.00% test_error 90.60%\n",
      "503 acc_train_loss 12608.14 acc_train_error 90.00% test_error 90.60%\n",
      "504 acc_train_loss 12608.14 acc_train_error 90.00% test_error 90.60%\n",
      "505 acc_train_loss 12608.15 acc_train_error 90.00% test_error 90.60%\n",
      "506 acc_train_loss 12608.15 acc_train_error 90.00% test_error 90.50%\n",
      "507 acc_train_loss 12608.15 acc_train_error 90.00% test_error 90.60%\n",
      "508 acc_train_loss 12608.16 acc_train_error 90.10% test_error 90.60%\n",
      "509 acc_train_loss 12608.16 acc_train_error 90.00% test_error 90.60%\n",
      "510 acc_train_loss 12608.17 acc_train_error 90.00% test_error 90.60%\n",
      "511 acc_train_loss 12608.17 acc_train_error 90.00% test_error 90.60%\n",
      "512 acc_train_loss 12608.17 acc_train_error 90.00% test_error 90.60%\n",
      "513 acc_train_loss 12608.17 acc_train_error 90.00% test_error 90.60%\n",
      "514 acc_train_loss 12608.18 acc_train_error 89.90% test_error 90.60%\n",
      "515 acc_train_loss 12608.18 acc_train_error 90.00% test_error 90.70%\n",
      "516 acc_train_loss 12608.19 acc_train_error 89.90% test_error 90.60%\n",
      "517 acc_train_loss 12608.19 acc_train_error 90.00% test_error 90.60%\n",
      "518 acc_train_loss 12608.20 acc_train_error 90.00% test_error 90.60%\n",
      "519 acc_train_loss 12608.20 acc_train_error 90.00% test_error 90.60%\n",
      "520 acc_train_loss 12608.20 acc_train_error 90.00% test_error 90.60%\n",
      "521 acc_train_loss 12608.20 acc_train_error 90.00% test_error 90.60%\n",
      "522 acc_train_loss 12608.20 acc_train_error 90.00% test_error 90.50%\n",
      "523 acc_train_loss 12608.21 acc_train_error 89.90% test_error 90.40%\n",
      "524 acc_train_loss 12608.21 acc_train_error 90.00% test_error 90.60%\n",
      "525 acc_train_loss 12608.21 acc_train_error 90.00% test_error 90.60%\n",
      "526 acc_train_loss 12608.21 acc_train_error 90.00% test_error 90.60%\n",
      "527 acc_train_loss 12608.22 acc_train_error 90.00% test_error 90.70%\n",
      "528 acc_train_loss 12608.22 acc_train_error 90.00% test_error 90.60%\n",
      "529 acc_train_loss 12608.22 acc_train_error 90.00% test_error 90.60%\n",
      "530 acc_train_loss 12608.22 acc_train_error 90.00% test_error 90.60%\n",
      "531 acc_train_loss 12608.23 acc_train_error 90.00% test_error 90.60%\n",
      "532 acc_train_loss 12608.23 acc_train_error 90.10% test_error 90.60%\n",
      "533 acc_train_loss 12608.24 acc_train_error 90.10% test_error 90.70%\n",
      "534 acc_train_loss 12608.24 acc_train_error 90.00% test_error 90.60%\n",
      "535 acc_train_loss 12608.24 acc_train_error 90.00% test_error 90.60%\n",
      "536 acc_train_loss 12608.24 acc_train_error 90.00% test_error 90.70%\n",
      "537 acc_train_loss 12608.25 acc_train_error 90.10% test_error 90.60%\n",
      "538 acc_train_loss 12608.25 acc_train_error 90.00% test_error 90.60%\n",
      "539 acc_train_loss 12608.25 acc_train_error 90.00% test_error 90.60%\n",
      "540 acc_train_loss 12608.25 acc_train_error 89.90% test_error 90.60%\n",
      "541 acc_train_loss 12608.26 acc_train_error 89.90% test_error 90.60%\n",
      "542 acc_train_loss 12608.26 acc_train_error 90.00% test_error 90.60%\n",
      "543 acc_train_loss 12608.26 acc_train_error 90.00% test_error 90.60%\n",
      "544 acc_train_loss 12608.27 acc_train_error 90.00% test_error 90.60%\n",
      "545 acc_train_loss 12608.27 acc_train_error 90.00% test_error 90.60%\n",
      "546 acc_train_loss 12608.27 acc_train_error 90.00% test_error 90.60%\n",
      "547 acc_train_loss 12608.27 acc_train_error 90.00% test_error 90.60%\n",
      "548 acc_train_loss 12608.28 acc_train_error 90.00% test_error 90.60%\n",
      "549 acc_train_loss 12608.28 acc_train_error 90.00% test_error 90.60%\n",
      "550 acc_train_loss 12608.28 acc_train_error 90.00% test_error 90.60%\n",
      "551 acc_train_loss 12608.29 acc_train_error 90.00% test_error 90.50%\n",
      "552 acc_train_loss 12608.29 acc_train_error 90.00% test_error 90.60%\n",
      "553 acc_train_loss 12608.29 acc_train_error 89.90% test_error 90.60%\n",
      "554 acc_train_loss 12608.29 acc_train_error 90.00% test_error 90.50%\n",
      "555 acc_train_loss 12608.29 acc_train_error 90.00% test_error 90.60%\n",
      "556 acc_train_loss 12608.30 acc_train_error 90.00% test_error 90.60%\n",
      "557 acc_train_loss 12608.30 acc_train_error 90.00% test_error 90.60%\n",
      "558 acc_train_loss 12608.30 acc_train_error 90.00% test_error 90.60%\n",
      "559 acc_train_loss 12608.31 acc_train_error 90.00% test_error 90.60%\n",
      "560 acc_train_loss 12608.31 acc_train_error 90.00% test_error 90.60%\n",
      "561 acc_train_loss 12608.31 acc_train_error 90.00% test_error 90.60%\n",
      "562 acc_train_loss 12608.31 acc_train_error 90.00% test_error 90.60%\n",
      "563 acc_train_loss 12608.31 acc_train_error 90.00% test_error 90.60%\n",
      "564 acc_train_loss 12608.31 acc_train_error 90.00% test_error 90.60%\n",
      "565 acc_train_loss 12608.32 acc_train_error 90.00% test_error 90.60%\n",
      "566 acc_train_loss 12608.32 acc_train_error 90.00% test_error 90.60%\n",
      "567 acc_train_loss 12608.32 acc_train_error 89.90% test_error 90.60%\n",
      "568 acc_train_loss 12608.32 acc_train_error 90.00% test_error 90.60%\n",
      "569 acc_train_loss 12608.33 acc_train_error 90.00% test_error 90.60%\n",
      "570 acc_train_loss 12608.33 acc_train_error 89.90% test_error 90.60%\n",
      "571 acc_train_loss 12608.33 acc_train_error 90.00% test_error 90.60%\n",
      "572 acc_train_loss 12608.33 acc_train_error 89.90% test_error 90.60%\n",
      "573 acc_train_loss 12608.34 acc_train_error 90.00% test_error 90.60%\n",
      "574 acc_train_loss 12608.34 acc_train_error 90.00% test_error 90.60%\n",
      "575 acc_train_loss 12608.34 acc_train_error 90.00% test_error 90.70%\n",
      "576 acc_train_loss 12608.34 acc_train_error 90.00% test_error 90.60%\n",
      "577 acc_train_loss 12608.35 acc_train_error 90.00% test_error 90.60%\n",
      "578 acc_train_loss 12608.35 acc_train_error 90.00% test_error 90.60%\n",
      "579 acc_train_loss 12608.35 acc_train_error 90.00% test_error 90.50%\n",
      "580 acc_train_loss 12608.36 acc_train_error 90.00% test_error 90.60%\n",
      "581 acc_train_loss 12608.36 acc_train_error 90.00% test_error 90.60%\n",
      "582 acc_train_loss 12608.36 acc_train_error 90.00% test_error 90.60%\n",
      "583 acc_train_loss 12608.36 acc_train_error 89.90% test_error 90.40%\n",
      "584 acc_train_loss 12608.37 acc_train_error 90.00% test_error 90.60%\n",
      "585 acc_train_loss 12608.37 acc_train_error 90.00% test_error 90.70%\n",
      "586 acc_train_loss 12608.37 acc_train_error 89.90% test_error 90.70%\n",
      "587 acc_train_loss 12608.38 acc_train_error 90.00% test_error 90.60%\n",
      "588 acc_train_loss 12608.38 acc_train_error 90.00% test_error 90.60%\n",
      "589 acc_train_loss 12608.38 acc_train_error 90.00% test_error 90.60%\n",
      "590 acc_train_loss 12608.39 acc_train_error 90.00% test_error 90.60%\n",
      "591 acc_train_loss 12608.39 acc_train_error 90.00% test_error 90.60%\n",
      "592 acc_train_loss 12608.39 acc_train_error 90.00% test_error 90.60%\n",
      "593 acc_train_loss 12608.40 acc_train_error 90.10% test_error 90.60%\n",
      "594 acc_train_loss 12608.40 acc_train_error 90.00% test_error 90.70%\n",
      "595 acc_train_loss 12608.41 acc_train_error 90.00% test_error 90.60%\n",
      "596 acc_train_loss 12608.41 acc_train_error 90.00% test_error 90.60%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "597 acc_train_loss 12608.41 acc_train_error 90.00% test_error 90.60%\n",
      "598 acc_train_loss 12608.41 acc_train_error 90.00% test_error 90.60%\n",
      "599 acc_train_loss 12608.42 acc_train_error 89.90% test_error 90.60%\n",
      "600 acc_train_loss 12608.42 acc_train_error 90.00% test_error 90.50%\n",
      "601 acc_train_loss 12608.43 acc_train_error 90.00% test_error 90.60%\n",
      "602 acc_train_loss 12608.43 acc_train_error 89.90% test_error 90.60%\n",
      "603 acc_train_loss 12608.43 acc_train_error 90.00% test_error 90.60%\n",
      "604 acc_train_loss 12608.43 acc_train_error 90.00% test_error 90.60%\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-f7ffc272cc94>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_train_samples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;31m#forward prop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0mx0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforward_pass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mw1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_input\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#takes class with maximum prob(x2, returns prob)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-b14ed862b008>\u001b[0m in \u001b[0;36mforward_pass\u001b[0;34m(w1, b1, w2, b2, x)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward_pass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mw1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mx0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0ms1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mw1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mb1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mx1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msigma\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0ms2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mw2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mx1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mb2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for k in range(1000):\n",
    "    \n",
    "    accumulated_loss = 0\n",
    "    n_train_errors = 0\n",
    "    \n",
    "    \"\"\"\n",
    "    Performing 1,000 gradient steps with a step size equal to 0.1 divided\n",
    "    by the number of training samples(variable--steps)\n",
    "    First reset the tensors to zero for summing up the gradients and doing\n",
    "    a forward and backward pass for each training example\n",
    "    \"\"\"\n",
    "    dl_dw1.zero_()\n",
    "    dl_db1.zero_()\n",
    "    dl_dw2.zero_()\n",
    "    dl_db2.zero_()\n",
    "    \n",
    "    for n in range(n_train_samples):\n",
    "        #forward prop\n",
    "        x0, s1, x1, s2, x2 = forward_pass(w1, b1, w2, b2, train_input[n])\n",
    "        \n",
    "        pred = x2.max(0)[1].item() #takes class with maximum prob(x2, returns prob)\n",
    "        if train_target[n, pred] < 0.5:\n",
    "            n_train_errors = n_train_errors + 1\n",
    "        accumulated_loss = accumulated_loss + loss(x2, train_target[n])\n",
    "            \n",
    "    \n",
    "        #backward prop\n",
    "        backward_pass(w1, b1, w2, b2,\n",
    "                      train_target[n],\n",
    "                      x0, s1, x1, s2, x2,\n",
    "                      dl_dw1, dl_db1, dl_dw2, dl_db2)\n",
    "    \n",
    "        #update rule\n",
    "    w1 = w1 - step * dl_dw1\n",
    "    b1 = b1 - step * dl_db1\n",
    "    w2 = w2 - step * dl_dw2\n",
    "    b2 = b2 - step * dl_db2\n",
    "    \n",
    "     # Test error\n",
    "\n",
    "    n_test_errors = 0\n",
    "\n",
    "    for n in range(test_input.size(0)):\n",
    "        _, _, _, _, x2 = forward_pass(w1, b1, w2, b2, test_input[n])\n",
    "\n",
    "        pred = x2.max(0)[1].item()\n",
    "        if test_target[n, pred] < 0.5: n_test_errors = n_test_errors + 1\n",
    "\n",
    "    print('{:d} acc_train_loss {:.02f} acc_train_error {:.02f}% test_error {:.02f}%'\n",
    "          .format(k,\n",
    "                  accumulated_loss,\n",
    "                  (100 * n_train_errors) / train_input.size(0),\n",
    "                  (100 * n_test_errors) / test_input.size(0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}