examples/hyper_parameters_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-27T19:28:59.644293Z",
     "start_time": "2022-04-27T19:28:57.300357Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from orbit.models import LGT, DLT\n",
    "from orbit.utils.dataset import load_iclaims\n",
    "\n",
    "from orbit.utils.params_tuning import grid_search_orbit\n",
    "from orbit.diagnostics.metrics import smape, mape, wmape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-27T19:29:01.763630Z",
     "start_time": "2022-04-27T19:29:00.375187Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(443, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>week</th>\n",
       "      <th>claims</th>\n",
       "      <th>trend.unemploy</th>\n",
       "      <th>trend.filling</th>\n",
       "      <th>trend.job</th>\n",
       "      <th>sp500</th>\n",
       "      <th>vix</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2010-01-03</td>\n",
       "      <td>13.386595</td>\n",
       "      <td>0.219882</td>\n",
       "      <td>-0.318452</td>\n",
       "      <td>0.117500</td>\n",
       "      <td>-0.417633</td>\n",
       "      <td>0.122654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2010-01-10</td>\n",
       "      <td>13.624218</td>\n",
       "      <td>0.219882</td>\n",
       "      <td>-0.194838</td>\n",
       "      <td>0.168794</td>\n",
       "      <td>-0.425480</td>\n",
       "      <td>0.110445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2010-01-17</td>\n",
       "      <td>13.398741</td>\n",
       "      <td>0.236143</td>\n",
       "      <td>-0.292477</td>\n",
       "      <td>0.117500</td>\n",
       "      <td>-0.465229</td>\n",
       "      <td>0.532339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2010-01-24</td>\n",
       "      <td>13.137549</td>\n",
       "      <td>0.203353</td>\n",
       "      <td>-0.194838</td>\n",
       "      <td>0.106918</td>\n",
       "      <td>-0.481751</td>\n",
       "      <td>0.428645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2010-01-31</td>\n",
       "      <td>13.196760</td>\n",
       "      <td>0.134360</td>\n",
       "      <td>-0.242466</td>\n",
       "      <td>0.074483</td>\n",
       "      <td>-0.488929</td>\n",
       "      <td>0.487404</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        week     claims  trend.unemploy  trend.filling  trend.job     sp500  \\\n",
       "0 2010-01-03  13.386595        0.219882      -0.318452   0.117500 -0.417633   \n",
       "1 2010-01-10  13.624218        0.219882      -0.194838   0.168794 -0.425480   \n",
       "2 2010-01-17  13.398741        0.236143      -0.292477   0.117500 -0.465229   \n",
       "3 2010-01-24  13.137549        0.203353      -0.194838   0.106918 -0.481751   \n",
       "4 2010-01-31  13.196760        0.134360      -0.242466   0.074483 -0.488929   \n",
       "\n",
       "        vix  \n",
       "0  0.122654  \n",
       "1  0.110445  \n",
       "2  0.532339  \n",
       "3  0.428645  \n",
       "4  0.487404  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data = load_iclaims()\n",
    "data = raw_data.copy()\n",
    "\n",
    "print(data.shape)\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LGT Tuning Exampe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"alert alert-info\">\n",
    "\n",
    "To have a consistent outlook here, users need to make sure an updated version of `ipywidgets` is installed.\n",
    "\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-27T19:29:01.768263Z",
     "start_time": "2022-04-27T19:29:01.765291Z"
    }
   },
   "outputs": [],
   "source": [
    "lgt = LGT(\n",
    "    date_col='week',\n",
    "    response_col='claims',\n",
    "    seasonality=52,\n",
    "    estimator='stan-map',\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-27T19:29:01.772806Z",
     "start_time": "2022-04-27T19:29:01.770683Z"
    }
   },
   "outputs": [],
   "source": [
    "param_grid = {'level_sm_input': [0.3, 0.5, 0.8],\n",
    "              'seasonality_sm_input': [0.3, 0.5, 0.8]}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Backtest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-27T19:29:12.390487Z",
     "start_time": "2022-04-27T19:29:07.961263Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e06205616c0482fa2743fe94bb95ddb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "best_params, tuned_df = grid_search_orbit(\n",
    "    param_grid,                           \n",
    "    model=lgt,                           \n",
    "    df=data,                           \n",
    "    min_train_len=100,                         \n",
    "    incremental_len=100,                    \n",
    "    forecast_len=20,                        \n",
    "    eval_method='backtest',                           \n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-27T19:29:12.402658Z",
     "start_time": "2022-04-27T19:29:12.393152Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>level_sm_input</th>\n",
       "      <th>seasonality_sm_input</th>\n",
       "      <th>metrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.3</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.006624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.006012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.005633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.007426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.006844</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   level_sm_input  seasonality_sm_input   metrics\n",
       "0             0.3                   0.3  0.006624\n",
       "1             0.3                   0.5  0.006012\n",
       "2             0.3                   0.8  0.005633\n",
       "3             0.5                   0.3  0.007426\n",
       "4             0.5                   0.5  0.006844"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tuned_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-27T19:29:12.408718Z",
     "start_time": "2022-04-27T19:29:12.405208Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'level_sm_input': 0.3, 'seasonality_sm_input': 0.8}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BIC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Users can also tune the model using BIC by specifying `eval_method='bic'`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-27T19:29:13.604485Z",
     "start_time": "2022-04-27T19:29:12.411236Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51228c5c98fa4a6f9e1862a1784a3947",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "best_params, tuned_df = grid_search_orbit(\n",
    "    param_grid,                      \n",
    "    model=lgt,                       \n",
    "    df=data,     \n",
    "    min_train_len=100,             \n",
    "    incremental_len=100,            \n",
    "    forecast_len=20,               \n",
    "    eval_method='bic',                             \n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-27T19:29:18.938741Z",
     "start_time": "2022-04-27T19:29:18.934731Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'level_sm_input': 0.3, 'seasonality_sm_input': 0.8}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DLT Tuning Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-21T23:19:24.456418Z",
     "start_time": "2022-04-21T23:19:24.451866Z"
    }
   },
   "outputs": [],
   "source": [
    "dlt = DLT(\n",
    "    date_col='week',\n",
    "    response_col='claims',\n",
    "    regressor_col=['trend.unemploy', 'trend.filling', 'trend.job'],\n",
    "    seasonality=52,\n",
    "    estimator='stan-map',\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-21T23:19:25.849608Z",
     "start_time": "2022-04-21T23:19:25.846770Z"
    }
   },
   "outputs": [],
   "source": [
    "param_grid = {'damped_factor': [0.3, 0.5, 0.8],\n",
    "              'slope_sm_input': [0.3, 0.5, 0.8]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-21T23:19:38.184082Z",
     "start_time": "2022-04-21T23:19:33.087777Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34b97dcb7f2e4ca5b91f088d6332c650",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "best_params, tuned_df = grid_search_orbit(\n",
    "    param_grid,\n",
    "    model=dlt,\n",
    "    df=data,\n",
    "    min_train_len=100,                \n",
    "    incremental_len=100,                  \n",
    "    forecast_len=20,                     \n",
    "    eval_method='backtest',                 \n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-21T23:19:38.194110Z",
     "start_time": "2022-04-21T23:19:38.186143Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>damped_factor</th>\n",
       "      <th>slope_sm_input</th>\n",
       "      <th>metrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.3</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.005776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.005826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.005813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.005788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.005637</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   damped_factor  slope_sm_input   metrics\n",
       "0            0.3             0.3  0.005776\n",
       "1            0.3             0.5  0.005826\n",
       "2            0.3             0.8  0.005813\n",
       "3            0.5             0.3  0.005788\n",
       "4            0.5             0.5  0.005637"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tuned_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-21T23:19:38.200572Z",
     "start_time": "2022-04-21T23:19:38.197496Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'damped_factor': 0.5, 'slope_sm_input': 0.5}]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_params"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "orbit38",
   "language": "python",
   "name": "orbit38"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}