overfitting_probelms_and_regularization.ipynb

{
 "cells":[
  {
   "cell_type":"markdown",
   "source":[
    "**This notebook covers work on problems related to overfitting models and how regularization helps achieve more generalization and move away from high variance**"
   ],
   "attachments":{
    
   },
   "metadata":{
    "datalore":{
     "node_id":"Pk4qqth7Pk28r7fx7DWH0m",
     "type":"MD",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# let's start by importing the required libraries\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import copy, math"
   ],
   "execution_count":11,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"Jz07m1vzcwSiZ4pP0wRaB5",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# let's start by defining the linear regression prediciton function\n",
    "\n",
    "def lin_pred_y(x,w,b):\n",
    "    pred_y = np.dot(x,w) + b\n",
    "    return pred_y"
   ],
   "execution_count":21,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"W647m0FDWBYPApdNX7Ig1R",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the sigmoid function \n",
    "\n",
    "def sig_pred_y(z):\n",
    "    pred_y = 1\/(1+np.exp(-z))\n",
    "    return pred_y"
   ],
   "execution_count":4,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"SfD7xlBTMJJWwBq4sizSgC",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the cost function for linear regression\n",
    "\n",
    "def cost_lin_reg(x,y,w,b):\n",
    "\n",
    "    size_x = x.shape[0]\n",
    "    features_x = x.shape[1]\n",
    "    cost = 0\n",
    "    for i in range(size_x):\n",
    "        error = lin_pred_y(x[i],w,b) - y[i]\n",
    "        cost += np.square(error)\n",
    "\n",
    "    cost = cost\/(2*size_x)\n",
    "\n",
    "    return cost"
   ],
   "execution_count":5,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"HirY2eb8Bhx3ifUgcFS8Wq",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the cost function for logistic regression\n",
    "\n",
    "def cost_log_reg(x,y,w,b):\n",
    "\n",
    "    size_x = x.shape[0]\n",
    "    cost=0\n",
    "\n",
    "    for i in range(size_x):\n",
    "        pred_y = sig_pred_y(lin_pred_y(x[i],w,b))\n",
    "\n",
    "        loss  = -y[i] * np.log(pred_y) - (1-y[i])*np.log(1-pred_y)\n",
    "            \n",
    "        cost += loss\n",
    "            \n",
    "    cost = cost\/size_x\n",
    "    \n",
    "    return cost"
   ],
   "execution_count":6,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"FuLqKO6pH7wUk1mblz0XEq",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the compute gradient function for linear regression\n",
    "\n",
    "def compute_lin_gradient_fn(x,y,w,b):\n",
    "\n",
    "    size_x = x.shape[0]\n",
    "    features_x = x.shape[1]\n",
    "    dj_dw = np.zeros(features_x)\n",
    "    dj_db = 0\n",
    "\n",
    "    for i in range(size_x):\n",
    "\n",
    "        pred_y = lin_pred_y(x[i],w,b)\n",
    "        error = pred_y - y[i]\n",
    "        for j in range(features_x):\n",
    "            dj_dw[j] += error * x[i,j]\n",
    "        dj_db += error\n",
    "\n",
    "    dj_dw = dj_dw\/size_x\n",
    "    dj_db = dj_db\/size_x\n",
    "\n",
    "    return dj_dw, dj_db"
   ],
   "execution_count":9,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"Oz7HLXRwTFqsyMbuhSNdpY",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the function to compute gradient function for logistic regression\n",
    "\n",
    "def compute_log_gradient_fn(x,y,w,b):\n",
    "\n",
    "    size_x = x.shape[0]\n",
    "    features_x = x.shape[1]\n",
    "    dj_dw = np.zeros(features_x)\n",
    "    dj_db = 0\n",
    "\n",
    "    for i in range(size_x):\n",
    "\n",
    "        pred_y = sig_pred_y(lin_pred_y(x[i],w,b))\n",
    "        error = pred_y - y[i]\n",
    "        for j in range(features_x):\n",
    "            dj_dw[j] += error * x[i,j]\n",
    "        dj_db += error\n",
    "\n",
    "    dj_dw = dj_dw\/size_x\n",
    "    dj_db = dj_db\/size_x\n",
    "\n",
    "    return dj_dw, dj_db"
   ],
   "execution_count":53,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"leixz6AIdbJPUHv89BsTVD",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the function to run gradient descent for linear regression\n",
    "\n",
    "def run_gradient_descent_lin_reg(x,y,w,b,n,a):\n",
    "\n",
    "    w_tmp = copy.deepcopy(w)\n",
    "    b_tmp = b\n",
    "    size_x = x.shape[0]\n",
    "    cost_hist = []\n",
    "    for i in range(n):\n",
    "        dj_dw, dj_db = compute_lin_gradient_fn(x,y,w_tmp,b_tmp)\n",
    "        w_tmp = w_tmp - a*dj_dw\n",
    "        b_tmp = b_tmp - a*dj_db\n",
    "        cost = cost_lin_reg(x,y,w,b)\n",
    "        cost_hist.append(cost)\n",
    "        \n",
    "    return w_tmp, b_tmp, cost_hist"
   ],
   "execution_count":14,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"48lXhMKVFSXyhTWGzZNVQR",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the function to run gradient descent for logistic regression\n",
    "\n",
    "def run_gradient_descent_log_reg(x,y,w,b,n,a):\n",
    "\n",
    "    w_tmp = copy.deepcopy(w)\n",
    "    b_tmp = b\n",
    "    size_x = x.shape[0]\n",
    "    cost_hist = []\n",
    "    for i in range(n):\n",
    "        dj_dw, dj_db = compute_log_gradient_fn(x,y,w_tmp,b_tmp)\n",
    "        w_tmp = w_tmp - a*dj_dw\n",
    "        b_tmp = b_tmp - a*dj_db\n",
    "        cost = cost_lin_reg(x,y,w,b)\n",
    "        cost_hist.append(cost)\n",
    "        \n",
    "    return w_tmp, b_tmp, cost_hist"
   ],
   "execution_count":15,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"t0gVZPbmqPJKhprf73Jf72",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the regularized cost function for linear regression\n",
    "\n",
    "# in order to regularize cost function we add Y\/2m * (sum(square(weight params)))\n",
    "# Y here is lambra is the degree of regularization\n",
    "\n",
    "def reg_cost_lin_reg(x,y,w,b,Y=1.0):\n",
    "\n",
    "    size_x = x.shape[0]\n",
    "    features_x = x.shape[1]\n",
    "    cost = 0\n",
    "    for i in range(size_x):\n",
    "        error = lin_pred_y(x[i],w,b) - y[i]\n",
    "        cost += np.square(error)\n",
    "\n",
    "    cost = cost\/(2*size_x)\n",
    "\n",
    "    # let's calculate regularization here\n",
    "    reg_cost = 0\n",
    "    for j in range(features_x):\n",
    "        reg_cost+=np.square(w[j])\n",
    "\n",
    "    reg_cost = (Y\/(2*size_x))*reg_cost\n",
    "    total_cost = cost + reg_cost\n",
    "    return total_cost"
   ],
   "execution_count":28,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"sQwZJZyatyF3N0IJsnHYuX",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the regularized cost function for logistic regression \n",
    "# in order to regularize cost function we add Y\/2m * (sum(square(weight params)))\n",
    "# Y here is lambra is the degree of regularization\n",
    "\n",
    "def reg_cost_log_reg(x,y,w,b,Y=1):\n",
    "\n",
    "    size_x = x.shape[0]\n",
    "    features_x = x.shape[1]\n",
    "    cost=0\n",
    "\n",
    "    for i in range(size_x):\n",
    "        pred_y = sig_pred_y(lin_pred_y(x[i],w,b))\n",
    "\n",
    "        loss  = -y[i] * np.log(pred_y) - (1-y[i])*np.log(1-pred_y)\n",
    "            \n",
    "        cost += loss\n",
    "            \n",
    "    cost = cost\/size_x\n",
    "    \n",
    "    # let's calculate regularization here\n",
    "    reg_cost = 0\n",
    "    for j in range(features_x):\n",
    "        reg_cost+=np.square(w[j])\n",
    "\n",
    "    reg_cost = (Y\/(2*size_x))*reg_cost\n",
    "    total_cost = cost + reg_cost\n",
    "    return total_cost"
   ],
   "execution_count":29,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"bkjFNjWOZ1KdBh7mxjImGX",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# let's test our regularize cost function for linear regression\n",
    "\n",
    "np.random.seed(1)\n",
    "X_tmp = np.random.rand(5,6)\n",
    "y_tmp = np.array([0,1,0,1,0])\n",
    "w_tmp = np.random.rand(X_tmp.shape[1]).reshape(-1,)-0.5\n",
    "b_tmp = 0.5\n",
    "lambda_tmp = 0.7\n",
    "cost_tmp = reg_cost_lin_reg(X_tmp, y_tmp, w_tmp, b_tmp, lambda_tmp)\n",
    "\n",
    "print(\"Regularized cost:\", cost_tmp)"
   ],
   "execution_count":30,
   "outputs":[
    {
     "name":"stdout",
     "text":[
      "Regularized cost: 0.07917239320214277\n"
     ],
     "output_type":"stream"
    }
   ],
   "metadata":{
    "datalore":{
     "node_id":"3yHkbWUaT2CbogXRXj3iVQ",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's test the regularized cost function for logistic regresion\n",
    "\n",
    "np.random.seed(1)\n",
    "X_tmp = np.random.rand(5,6)\n",
    "y_tmp = np.array([0,1,0,1,0])\n",
    "w_tmp = np.random.rand(X_tmp.shape[1]).reshape(-1,)-0.5\n",
    "b_tmp = 0.5\n",
    "lambda_tmp = 0.7\n",
    "cost_tmp = reg_cost_log_reg(X_tmp, y_tmp, w_tmp, b_tmp, lambda_tmp)\n",
    "\n",
    "print(\"Regularized cost:\", cost_tmp)"
   ],
   "execution_count":31,
   "outputs":[
    {
     "name":"stdout",
     "text":[
      "Regularized cost: 0.6850849138741673\n"
     ],
     "output_type":"stream"
    }
   ],
   "metadata":{
    "datalore":{
     "node_id":"CXU0OBZisjkwIDEZbaW4X0",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the compute gradient function for regularized linear regression\n",
    "# when deriving derivative the derivative of regularization becomes Y\/m*w atj (this is when only applying it to w)\n",
    "def compute_lin_gradient_reg_fn(x,y,w,b,Y=1.0):\n",
    "\n",
    "    size_x = x.shape[0]\n",
    "    features_x = x.shape[1]\n",
    "    dj_dw = np.zeros(features_x)\n",
    "    dj_db = 0\n",
    "\n",
    "    for i in range(size_x):\n",
    "\n",
    "        pred_y = lin_pred_y(x[i],w,b)\n",
    "        error = pred_y - y[i]\n",
    "        for j in range(features_x):\n",
    "            dj_dw[j] += error * x[i,j]\n",
    "        dj_db += error\n",
    "\n",
    "    dj_dw = dj_dw\/size_x\n",
    "    dj_db = dj_db\/size_x\n",
    "\n",
    "    for j in range(features_x):\n",
    "        dj_dw[j] = dj_dw[j]+((Y\/size_x)*w[j])\n",
    "\n",
    "    return dj_db, dj_dw"
   ],
   "execution_count":46,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"rxYMEO7F5gmrB5jHLpyShR",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# let's put the regularized compute gradient derivatives function to test\n",
    "\n",
    "np.random.seed(1)\n",
    "X_tmp = np.random.rand(5,3)\n",
    "y_tmp = np.array([0,1,0,1,0])\n",
    "w_tmp = np.random.rand(X_tmp.shape[1])\n",
    "b_tmp = 0.5\n",
    "lambda_tmp = 0.7\n",
    "dj_db_tmp, dj_dw_tmp =  compute_lin_gradient_reg_fn(X_tmp, y_tmp, w_tmp, b_tmp, lambda_tmp)\n",
    "\n",
    "print(f\"dj_db: {dj_db_tmp}\", )\n",
    "print(f\"Regularized dj_dw:\\n {dj_dw_tmp.tolist()}\", )"
   ],
   "execution_count":47,
   "outputs":[
    {
     "name":"stdout",
     "text":[
      "dj_db: 0.6648774569425726\n",
      "Regularized dj_dw:\n",
      " [0.29653214748822276, 0.4911679625918033, 0.21645877535865857]\n"
     ],
     "output_type":"stream"
    }
   ],
   "metadata":{
    "datalore":{
     "node_id":"KBOhRz7Ui4RjhxS1ab8ddp",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "# now let's define the regularized function to compute gradient function for logistic regression\n",
    "\n",
    "def compute_log_gradient_reg_fn(x,y,w,b,Y=1.):\n",
    "\n",
    "    size_x = x.shape[0]\n",
    "    features_x = x.shape[1]\n",
    "    dj_dw = np.zeros(features_x)\n",
    "    dj_db = 0\n",
    "\n",
    "    for i in range(size_x):\n",
    "\n",
    "        pred_y = sig_pred_y(lin_pred_y(x[i],w,b))\n",
    "        error = pred_y - y[i]\n",
    "        for j in range(features_x):\n",
    "            dj_dw[j] += error * x[i,j]\n",
    "        dj_db += error\n",
    "\n",
    "    dj_dw = dj_dw\/size_x\n",
    "    dj_db = dj_db\/size_x\n",
    "\n",
    "    for j in range(features_x):\n",
    "        dj_dw[j] = dj_dw[j]+((Y\/size_x)*w[j])\n",
    "\n",
    "    return dj_db, dj_dw"
   ],
   "execution_count":51,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"mBGIoORy2XMM1PFL4H5rAP",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    "np.random.seed(1)\n",
    "X_tmp = np.random.rand(5,3)\n",
    "y_tmp = np.array([0,1,0,1,0])\n",
    "w_tmp = np.random.rand(X_tmp.shape[1])\n",
    "b_tmp = 0.5\n",
    "lambda_tmp = 0.7\n",
    "dj_db_tmp, dj_dw_tmp =  compute_log_gradient_reg_fn(X_tmp, y_tmp, w_tmp, b_tmp, lambda_tmp)\n",
    "\n",
    "print(f\"dj_db: {dj_db_tmp}\", )\n",
    "print(f\"Regularized dj_dw:\\n {dj_dw_tmp.tolist()}\", )"
   ],
   "execution_count":52,
   "outputs":[
    {
     "name":"stdout",
     "text":[
      "dj_db: 0.341798994972791\n",
      "Regularized dj_dw:\n",
      " [0.17380012933994293, 0.32007507881566943, 0.10776313396851499]\n"
     ],
     "output_type":"stream"
    }
   ],
   "metadata":{
    "datalore":{
     "node_id":"xTlQnG4PsKmBeviDrVfzX3",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    
   ],
   "execution_count":null,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"vcP8cYO6q3LVCP2wvkEr0e",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  },
  {
   "cell_type":"code",
   "source":[
    
   ],
   "execution_count":null,
   "outputs":[
    
   ],
   "metadata":{
    "datalore":{
     "node_id":"dlJvOxcp9ezd5veZIppuuv",
     "type":"CODE",
     "hide_input_from_viewers":true,
     "hide_output_from_viewers":true
    }
   }
  }
 ],
 "metadata":{
  "kernelspec":{
   "display_name":"Python",
   "language":"python",
   "name":"python"
  },
  "datalore":{
   "computation_mode":"JUPYTER",
   "package_manager":"pip",
   "base_environment":"default",
   "packages":[
    
   ],
   "report_row_ids":[
    
   ],
   "version":3
  }
 },
 "nbformat":4,
 "nbformat_minor":4
}