Lecture 6 Simple two Linear layers with bias.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tensor = torch.tensor([0.1, 0.2, 0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tensor.reshape(1,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.1000, 0.2000, 0.3000]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(torch.nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(in_features=3, out_features=3, bias=True)\n",
    "        self.fc2 = torch.nn.Linear(in_features=3, out_features=1, bias=True)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        print(x)\n",
    "        print(x.shape)\n",
    "        x = self.fc1(x)\n",
    "        print(x)\n",
    "        print(x.shape)\n",
    "        x = self.fc2(x)\n",
    "        print(x)\n",
    "        print(x.shape)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Net()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "par = []\n",
    "for parameter in model.parameters():\n",
    "    par.append(parameter)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.optim import Optimizer\n",
    "class _RequiredParameter(object):\n",
    "    \"\"\"Singleton class representing a required parameter for an Optimizer.\"\"\"\n",
    "    def __repr__(self):\n",
    "        return \"<required parameter>\"\n",
    "\n",
    "required = _RequiredParameter()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class SGD(Optimizer):\n",
    "    r\"\"\"Implements stochastic gradient descent (optionally with momentum).\n",
    "    Nesterov momentum is based on the formula from\n",
    "    `On the importance of initialization and momentum in deep learning`__.\n",
    "    Args:\n",
    "        params (iterable): iterable of parameters to optimize or dicts defining\n",
    "            parameter groups\n",
    "        lr (float): learning rate\n",
    "        momentum (float, optional): momentum factor (default: 0)\n",
    "        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)\n",
    "        dampening (float, optional): dampening for momentum (default: 0)\n",
    "        nesterov (bool, optional): enables Nesterov momentum (default: False)\n",
    "    Example:\n",
    "        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)\n",
    "        >>> optimizer.zero_grad()\n",
    "        >>> loss_fn(model(input), target).backward()\n",
    "        >>> optimizer.step()\n",
    "    __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf\n",
    "    .. note::\n",
    "        The implementation of SGD with Momentum/Nesterov subtly differs from\n",
    "        Sutskever et. al. and implementations in some other frameworks.\n",
    "        Considering the specific case of Momentum, the update can be written as\n",
    "        .. math::\n",
    "                  v = \\rho * v + g \\\\\n",
    "                  p = p - lr * v\n",
    "        where p, g, v and :math:`\\rho` denote the parameters, gradient,\n",
    "        velocity, and momentum respectively.\n",
    "        This is in contrast to Sutskever et. al. and\n",
    "        other frameworks which employ an update of the form\n",
    "        .. math::\n",
    "             v = \\rho * v + lr * g \\\\\n",
    "             p = p - v\n",
    "        The Nesterov version is analogously modified.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, params, lr=required, momentum=0, dampening=0,\n",
    "                 weight_decay=0, nesterov=False):\n",
    "        if lr is not required and lr < 0.0:\n",
    "            raise ValueError(\"Invalid learning rate: {}\".format(lr))\n",
    "        if momentum < 0.0:\n",
    "            raise ValueError(\"Invalid momentum value: {}\".format(momentum))\n",
    "        if weight_decay < 0.0:\n",
    "            raise ValueError(\"Invalid weight_decay value: {}\".format(weight_decay))\n",
    "\n",
    "        defaults = dict(lr=lr, momentum=momentum, dampening=dampening,\n",
    "                        weight_decay=weight_decay, nesterov=nesterov)\n",
    "        if nesterov and (momentum <= 0 or dampening != 0):\n",
    "            raise ValueError(\"Nesterov momentum requires a momentum and zero dampening\")\n",
    "        super(SGD, self).__init__(params, defaults)\n",
    "\n",
    "    def __setstate__(self, state):\n",
    "        super(SGD, self).__setstate__(state)\n",
    "        for group in self.param_groups:\n",
    "            group.setdefault('nesterov', False)\n",
    "\n",
    "    def step(self, closure=None):\n",
    "        \"\"\"Performs a single optimization step.\n",
    "        Arguments:\n",
    "            closure (callable, optional): A closure that reevaluates the model\n",
    "                and returns the loss.\n",
    "        \"\"\"\n",
    "        loss = None\n",
    "        if closure is not None:\n",
    "            loss = closure()\n",
    "\n",
    "        for group in self.param_groups:\n",
    "            weight_decay = group['weight_decay']\n",
    "            momentum = group['momentum']\n",
    "            dampening = group['dampening']\n",
    "            nesterov = group['nesterov']\n",
    "\n",
    "            for p in group['params']:\n",
    "                if p.grad is None:\n",
    "                    continue\n",
    "                d_p = p.grad.data\n",
    "                if weight_decay != 0:\n",
    "                    d_p.add_(weight_decay, p.data)\n",
    "                if momentum != 0:\n",
    "                    param_state = self.state[p]\n",
    "                    if 'momentum_buffer' not in param_state:\n",
    "                        buf = param_state['momentum_buffer'] = torch.clone(d_p).detach()\n",
    "                    else:\n",
    "                        buf = param_state['momentum_buffer']\n",
    "                        buf.mul_(momentum).add_(1 - dampening, d_p)\n",
    "                    if nesterov:\n",
    "                        d_p = d_p.add(momentum, buf)\n",
    "                    else:\n",
    "                        d_p = buf\n",
    "\n",
    "                print('-'*15)\n",
    "                print('Parameter before update:',p)\n",
    "                print('Shape of parameter before update:',p.shape)\n",
    "                print('Gradient:',d_p)\n",
    "                print('Shape of the gradient:',d_p.shape)\n",
    "                print('X:',X)\n",
    "                print('Learning rate:',-group['lr'])\n",
    "                check = p.clone()\n",
    "                p.data.add_(-group['lr'], d_p)\n",
    "                print('Parameter of the tensor after update:', p)\n",
    "                print('Sanity check:', check +(-group['lr']) * d_p)\n",
    "                print('-'*15)\n",
    "\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer_detail = SGD(model.parameters(), lr=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# criterion = torch.nn.MSELoss()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.1000, 0.2000, 0.3000]])\n",
      "torch.Size([1, 3])\n",
      "tensor([[ 0.2572,  0.1864, -0.2532]], grad_fn=<AddmmBackward>)\n",
      "torch.Size([1, 3])\n",
      "tensor([[-0.5089]], grad_fn=<AddmmBackward>)\n",
      "torch.Size([1, 1])\n"
     ]
    }
   ],
   "source": [
    "output = model(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = output - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loss = criterion(1, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer_detail.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------\n",
      "Parameter before update: Parameter containing:\n",
      "tensor([[ 0.4722, -0.1605, -0.5165],\n",
      "        [-0.4895,  0.2991,  0.1742],\n",
      "        [ 0.0957, -0.1254,  0.2131]], requires_grad=True)\n",
      "Shape of parameter before update: torch.Size([3, 3])\n",
      "Gradient: tensor([[-0.0472, -0.0944, -0.1416],\n",
      "        [-0.0556, -0.1112, -0.1668],\n",
      "        [-0.0381, -0.0761, -0.1142]])\n",
      "Shape of the gradient: torch.Size([3, 3])\n",
      "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
      "Learning rate: -1\n",
      "Parameter of the tensor after update: Parameter containing:\n",
      "tensor([[ 0.5194, -0.0661, -0.3749],\n",
      "        [-0.4339,  0.4103,  0.3410],\n",
      "        [ 0.1337, -0.0492,  0.3273]], requires_grad=True)\n",
      "Sanity check: tensor([[ 0.5194, -0.0661, -0.3749],\n",
      "        [-0.4339,  0.4103,  0.3410],\n",
      "        [ 0.1337, -0.0492,  0.3273]], grad_fn=<AddBackward0>)\n",
      "---------------\n",
      "---------------\n",
      "Parameter before update: Parameter containing:\n",
      "tensor([ 0.3971,  0.1233, -0.3016], requires_grad=True)\n",
      "Shape of parameter before update: torch.Size([3])\n",
      "Gradient: tensor([-0.4719, -0.5560, -0.3805])\n",
      "Shape of the gradient: torch.Size([3])\n",
      "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
      "Learning rate: -1\n",
      "Parameter of the tensor after update: Parameter containing:\n",
      "tensor([0.8690, 0.6793, 0.0789], requires_grad=True)\n",
      "Sanity check: tensor([0.8690, 0.6793, 0.0789], grad_fn=<AddBackward0>)\n",
      "---------------\n",
      "---------------\n",
      "Parameter before update: Parameter containing:\n",
      "tensor([[-0.4719, -0.5560, -0.3805]], requires_grad=True)\n",
      "Shape of parameter before update: torch.Size([1, 3])\n",
      "Gradient: tensor([[ 0.2572,  0.1864, -0.2532]])\n",
      "Shape of the gradient: torch.Size([1, 3])\n",
      "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
      "Learning rate: -1\n",
      "Parameter of the tensor after update: Parameter containing:\n",
      "tensor([[-0.7292, -0.7424, -0.1273]], requires_grad=True)\n",
      "Sanity check: tensor([[-0.7292, -0.7424, -0.1273]], grad_fn=<AddBackward0>)\n",
      "---------------\n",
      "---------------\n",
      "Parameter before update: Parameter containing:\n",
      "tensor([-0.3802], requires_grad=True)\n",
      "Shape of parameter before update: torch.Size([1])\n",
      "Gradient: tensor([1.])\n",
      "Shape of the gradient: torch.Size([1])\n",
      "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
      "Learning rate: -1\n",
      "Parameter of the tensor after update: Parameter containing:\n",
      "tensor([-1.3802], requires_grad=True)\n",
      "Sanity check: tensor([-1.3802], grad_fn=<AddBackward0>)\n",
      "---------------\n"
     ]
    }
   ],
   "source": [
    "optimizer_detail.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "gradient = torch.tensor([-0.4719, -0.5560, -0.3805]).reshape(1,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0472, -0.0944, -0.1416],\n",
       "        [-0.0556, -0.1112, -0.1668],\n",
       "        [-0.0380, -0.0761, -0.1142]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradient.t().matmul(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-4.6789]], grad_fn=<SubBackward0>)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculations\n",
    "Going trough this from down to up because we are backpropagating:\n",
    "\n",
    "OUTPUT:\n",
    "\n",
    "```\n",
    "---------------\n",
    "---------------\n",
    "Parameter before update: Parameter containing:\n",
    "tensor([[ 0.4722, -0.1605, -0.5165],\n",
    "        [-0.4895,  0.2991,  0.1742],\n",
    "        [ 0.0957, -0.1254,  0.2131]], requires_grad=True)\n",
    "Shape of parameter before update: torch.Size([3, 3])\n",
    "Gradient: tensor([[-0.0472, -0.0944, -0.1416],\n",
    "        [-0.0556, -0.1112, -0.1668],\n",
    "        [-0.0381, -0.0761, -0.1142]])\n",
    "Shape of the gradient: torch.Size([3, 3])\n",
    "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
    "Learning rate: -1\n",
    "Parameter of the tensor after update: Parameter containing:\n",
    "tensor([[ 0.5194, -0.0661, -0.3749],\n",
    "        [-0.4339,  0.4103,  0.3410],\n",
    "        [ 0.1337, -0.0492,  0.3273]], requires_grad=True)\n",
    "Sanity check: tensor([[ 0.5194, -0.0661, -0.3749],\n",
    "        [-0.4339,  0.4103,  0.3410],\n",
    "        [ 0.1337, -0.0492,  0.3273]], grad_fn=<AddBackward0>)\n",
    "---------------\n",
    "---------------\n",
    "Parameter before update: Parameter containing:\n",
    "tensor([ 0.3971,  0.1233, -0.3016], requires_grad=True)\n",
    "Shape of parameter before update: torch.Size([3])\n",
    "Gradient: tensor([-0.4719, -0.5560, -0.3805])\n",
    "Shape of the gradient: torch.Size([3])\n",
    "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
    "Learning rate: -1\n",
    "Parameter of the tensor after update: Parameter containing:\n",
    "tensor([0.8690, 0.6793, 0.0789], requires_grad=True)\n",
    "Sanity check: tensor([0.8690, 0.6793, 0.0789], grad_fn=<AddBackward0>)\n",
    "---------------\n",
    "---------------\n",
    "Parameter before update: Parameter containing:\n",
    "tensor([[-0.4719, -0.5560, -0.3805]], requires_grad=True)\n",
    "Shape of parameter before update: torch.Size([1, 3])\n",
    "Gradient: tensor([[ 0.2572,  0.1864, -0.2532]])\n",
    "Shape of the gradient: torch.Size([1, 3])\n",
    "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
    "Learning rate: -1\n",
    "Parameter of the tensor after update: Parameter containing:\n",
    "tensor([[-0.7292, -0.7424, -0.1273]], requires_grad=True)\n",
    "Sanity check: tensor([[-0.7292, -0.7424, -0.1273]], grad_fn=<AddBackward0>)\n",
    "---------------\n",
    "---------------\n",
    "Parameter before update: Parameter containing:\n",
    "tensor([-0.3802], requires_grad=True)\n",
    "Shape of parameter before update: torch.Size([1])\n",
    "Gradient: tensor([1.])\n",
    "Shape of the gradient: torch.Size([1])\n",
    "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
    "Learning rate: -1\n",
    "Parameter of the tensor after update: Parameter containing:\n",
    "tensor([-1.3802], requires_grad=True)\n",
    "Sanity check: tensor([-1.3802], grad_fn=<AddBackward0>)\n",
    "---------------\n",
    "```\n",
    "\n",
    "Let's first consider:\n",
    "\n",
    "```\n",
    "---------------\n",
    "Parameter before update: Parameter containing:\n",
    "tensor([-0.3802], requires_grad=True)\n",
    "Shape of parameter before update: torch.Size([1])\n",
    "Gradient: tensor([1.])\n",
    "Shape of the gradient: torch.Size([1])\n",
    "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
    "Learning rate: -1\n",
    "Parameter of the tensor after update: Parameter containing:\n",
    "tensor([-1.3802], requires_grad=True)\n",
    "Sanity check: tensor([-1.3802], grad_fn=<AddBackward0>)\n",
    "---------------\n",
    "```\n",
    "\n",
    "This is obviously bias of the last layer or in other words second layer. We are taking derivative of wx+b w.r.t. b which is 1 which is fully confirmed by:\n",
    "\n",
    "`\n",
    "Gradient: tensor([1.])\n",
    "`\n",
    "\n",
    "Move backward to:\n",
    "\n",
    "```\n",
    "---------------\n",
    "Parameter before update: Parameter containing:\n",
    "tensor([[-0.4719, -0.5560, -0.3805]], requires_grad=True)\n",
    "Shape of parameter before update: torch.Size([1, 3])\n",
    "Gradient: tensor([[ 0.2572,  0.1864, -0.2532]])\n",
    "Shape of the gradient: torch.Size([1, 3])\n",
    "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
    "Learning rate: -1\n",
    "Parameter of the tensor after update: Parameter containing:\n",
    "tensor([[-0.7292, -0.7424, -0.1273]], requires_grad=True)\n",
    "Sanity check: tensor([[-0.7292, -0.7424, -0.1273]], grad_fn=<AddBackward0>)\n",
    "---------------\n",
    "```\n",
    "\n",
    "wx+b w.r.t. to to w is x and x is output from first layer.\n",
    "INPUT:\n",
    "\n",
    "```\n",
    "model(X)\n",
    "```\n",
    "\n",
    "OUTPUT:\n",
    "\n",
    "```\n",
    "tensor([[0.1000, 0.2000, 0.3000]])\n",
    "torch.Size([1, 3])\n",
    "tensor([[ 0.2572,  0.1864, -0.2532]], grad_fn=<AddmmBackward>)\n",
    "torch.Size([1, 3])\n",
    "tensor([[-0.5089]], grad_fn=<AddmmBackward>)\n",
    "torch.Size([1, 1])\n",
    "```\n",
    "\n",
    "We are particulary interested in:\n",
    "\n",
    "```\n",
    "tensor([[ 0.2572,  0.1864, -0.2532]], grad_fn=<AddmmBackward>)\n",
    "```\n",
    "\n",
    "which is equal to gradient of the tensor in 2nd layer or output from 1st layer:\n",
    "\n",
    "```\n",
    "Gradient: tensor([[ 0.2572,  0.1864, -0.2532]])\n",
    "```\n",
    "\n",
    "Now we are dealing with bias in the first layer. \n",
    "\n",
    "```\n",
    "---------------\n",
    "Parameter before update: Parameter containing:\n",
    "tensor([ 0.3971,  0.1233, -0.3016], requires_grad=True)\n",
    "Shape of parameter before update: torch.Size([3])\n",
    "Gradient: tensor([-0.4719, -0.5560, -0.3805])\n",
    "Shape of the gradient: torch.Size([3])\n",
    "X: tensor([[0.1000, 0.2000, 0.3000]])\n",
    "Learning rate: -1\n",
    "Parameter of the tensor after update: Parameter containing:\n",
    "tensor([0.8690, 0.6793, 0.0789], requires_grad=True)\n",
    "Sanity check: tensor([0.8690, 0.6793, 0.0789], grad_fn=<AddBackward0>)\n",
    "---------------\n",
    "```\n",
    "\n",
    "The derivative of wx+b w.r.t. b in the first layer is equal to 1. So we multiply by 1 derivative of the first layer by wx+b w.r.t. x. Pay attention to the x which is w.\n",
    "So, weight of the second layer is derivative of wx+b w.r.t. x in the second layer. Threfore, we multiply 1 * weight from previous layer(not updated, also pay attention to it) and it's equal to gradient of the bias in the first layer.\n",
    "\n",
    "```\n",
    "Gradient: tensor([-0.4719, -0.5560, -0.3805])\n",
    "```\n",
    "\n",
    "It's gradient of the bias which is equal to weight from previous layer. Have a look:\n",
    "\n",
    "```\n",
    "Parameter before update: Parameter containing:\n",
    "tensor([[-0.4719, -0.5560, -0.3805]], requires_grad=True)\n",
    "```\n",
    "\n",
    "And finally, first layer wx+b w.r.t. to w is equal x. We have to multiply by gradient from previous layer in other words by w.\n",
    "```\n",
    "---------------\n",
    "Parameter before update: Parameter containing:\n",
    "tensor([[-0.4719, -0.5560, -0.3805]], requires_grad=True)\n",
    "```\n",
    "\n",
    "INPUT:\n",
    "\n",
    "```\n",
    "gradient = torch.tensor([-0.4719, -0.5560, -0.3805]).reshape(1,3)\n",
    "gradient.t().matmul(X)\n",
    "```\n",
    "\n",
    "OUTPUT:\n",
    "\n",
    "```\n",
    "tensor([[-0.0472, -0.0944, -0.1416],\n",
    "        [-0.0556, -0.1112, -0.1668],\n",
    "        [-0.0380, -0.0761, -0.1142]])\n",
    "```\n",
    "\n",
    "which fully confirmed with autograd calculations:\n",
    "\n",
    "```\n",
    "Gradient: tensor([[-0.0472, -0.0944, -0.1416],\n",
    "        [-0.0556, -0.1112, -0.1668],\n",
    "        [-0.0381, -0.0761, -0.1142]])\n",
    "\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[-0.2185,  0.1295,  0.1802],\n",
       "        [-0.5210, -0.4478, -0.3567],\n",
       "        [-0.4256,  0.3492,  0.4206]], requires_grad=True)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "par[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.3222, -0.4164, -0.2208]], grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "layer1 = X.matmul(par[0].t()) + par[1]\n",
    "print(layer1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.5294]], grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "layer2 = layer1.matmul(par[2].t()) + par[3]\n",
    "print(layer2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.3222, -0.4164, -0.2208]], grad_fn=<AddmmBackward>)\n",
      "tensor([[-1.5294]], grad_fn=<AddmmBackward>)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[-1.5294]], grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Parameter containing:\n",
       " tensor([[-0.2185,  0.1295,  0.1802],\n",
       "         [-0.5210, -0.4478, -0.3567],\n",
       "         [-0.4256,  0.3492,  0.4206]], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([ 0.8732, -0.8030, -0.1054], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([[-0.8627,  1.0195,  0.0056]], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([-1.3930], requires_grad=True)]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "par"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}