Margin_six_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build 6 lightGBM models for Hierachical model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import lightgbm as lgb\n",
    "from datetime import date, timedelta\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import mean_absolute_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load\n",
    "data = pd.read_csv('df_egm.csv', parse_dates=['order_date'])\n",
    "# feature with no variance / duplicated feature\n",
    "data.drop(['store_id', 'is_liquidation', 'is_cancelled'], axis = 1, inplace = True)\n",
    "# duplicated\n",
    "data = data[~data['id'].duplicated()]\n",
    "# drop cancelled\n",
    "data = data[data['quantity_cancelled'] == 0]\n",
    "\n",
    "data['date'] = data['order_date'].dt.date\n",
    "data['cost'] = data['cost_product'] + data['cost_shipping'] + data['cost_other']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rtmask = (data['quantity_returned'] > 0) & (data['quantity_replaced'] == 0)\n",
    "rpmask = (data['quantity_replaced'] > 0) & (data['quantity_returned'] == 0)\n",
    "demask = (data['quantity_replaced'] == 0) & (data['quantity_returned'] == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prep_data(data, conts, cats, y_name):\n",
    "    # continuous features\n",
    "    ds = data[conts].copy()\n",
    "    # categorical features: code to 0 - (n-1)\n",
    "    for cat in cats:\n",
    "        ds[cat] = LabelEncoder().fit_transform(data[cat].copy())\n",
    "        ds.loc[data[cat].isnull(), cat] = np.nan\n",
    "    \n",
    "    # time\n",
    "    ds['month'] = data['order_date'].dt.month.copy() - 1\n",
    "    ds['dom'] = data['order_date'].dt.day.copy() - 1\n",
    "    ds['dow'] = data['order_date'].dt.dayofweek.copy()\n",
    "    cats.extend(['month', 'dow'])\n",
    "    \n",
    "    # train/val/test split\n",
    "    ds_train = ds[ds['date'] < date(2018, 6, 17)].copy()\n",
    "    ds_val = ds[(ds['date'] >= date(2018, 6, 17)) & (ds['date'] <= date(2018, 7, 16))].copy()\n",
    "    ds_test = ds[ds['date'] > date(2018, 7, 16)].copy()\n",
    "    ds_train.drop('date', axis = 1, inplace = True)\n",
    "    ds_val.drop('date', axis = 1, inplace = True)\n",
    "    ds_test.drop('date', axis = 1, inplace = True)\n",
    "    \n",
    "    # X/y split\n",
    "    X_train = ds_train.drop(y_name, axis = 1)\n",
    "    y_train = ds_train[y_name]\n",
    "    \n",
    "    X_val = ds_val.drop(y_name, axis = 1)\n",
    "    y_val = ds_val[y_name]\n",
    "    \n",
    "    X_test = ds_test.drop(y_name, axis = 1)\n",
    "    y_test = ds_test[y_name]\n",
    "    \n",
    "    return X_train, X_val, X_test, y_train, y_val, y_test, cats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(data, conts, cats, y_name):\n",
    "    X_train, X_val, X_test, y_train, y_val, y_test, cats = prep_data(data, conts, cats, y_name)\n",
    "    # set params\n",
    "    feat = X_train.columns.tolist()\n",
    "    params = {\n",
    "        'num_leaves': 31,\n",
    "        'objective': 'regression',\n",
    "        'min_data_in_leaf': 600,\n",
    "        'learning_rate': 0.05,\n",
    "        'feature_fraction': 0.8,\n",
    "        'bagging_fraction': 0.8,\n",
    "        'bagging_freq': 2,\n",
    "        'metric': 'l1',\n",
    "        'num_threads': 4\n",
    "    }\n",
    "\n",
    "    MAX_ROUNDS = 800\n",
    "    dtrain = lgb.Dataset(X_train, label = y_train, categorical_feature=cats)\n",
    "    dval = lgb.Dataset(X_val, label = y_val, reference=dtrain, categorical_feature=cats)\n",
    "    \n",
    "    bst = lgb.train(params, dtrain, num_boost_round=MAX_ROUNDS, valid_sets=[dtrain, dval],\n",
    "                    early_stopping_rounds=50, verbose_eval=50)\n",
    "    return bst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rev_name = 'revenue_net'\n",
    "rev_conts = ['date', 'sale_price', 'revenue_product', 'revenue_shipping', 'quantity_initial', 'revenue_net']\n",
    "rev_cats = ['shipping_speed_id', 'supplier_id', 'category_id', 'class_id', 'carrier_id', 'manufacturer_id', 'is_b2b', 'is_giftcard']\n",
    "\n",
    "cos_name = 'cost'\n",
    "cos_conts = ['date', 'sale_price', 'wholesale_price', 'quantity_initial', 'shipping_pred', 'cost']\n",
    "cos_cats = ['shipping_speed_id', 'supplier_id', 'category_id', 'class_id', 'carrier_id', 'manufacturer_id', 'is_b2b', 'is_giftcard']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prep_data2(data, conts, cats, y_name):\n",
    "    # continuous features\n",
    "    ds = data[conts].copy()\n",
    "    ds['id'] = data['id'].copy()\n",
    "    \n",
    "    # categorical features: code to 0 - (n-1)\n",
    "    for cat in cats:\n",
    "        ds[cat] = LabelEncoder().fit_transform(data[cat].copy())\n",
    "        ds.loc[data[cat].isnull(), cat] = np.nan\n",
    "    \n",
    "    # time\n",
    "    ds['month'] = data['order_date'].dt.month.copy() - 1\n",
    "    ds['dom'] = data['order_date'].dt.day.copy() - 1\n",
    "    ds['dow'] = data['order_date'].dt.dayofweek.copy()\n",
    "    cats.extend(['month', 'dow'])\n",
    "    \n",
    "    # train/val/test split\n",
    "    ds_train = ds[ds['date'] < date(2018, 6, 17)].copy()\n",
    "    ds_val = ds[(ds['date'] >= date(2018, 6, 17)) & (ds['date'] <= date(2018, 7, 16))].copy()\n",
    "    ds_test = ds[ds['date'] > date(2018, 7, 16)].copy()\n",
    "    ds_train.drop('date', axis = 1, inplace = True)\n",
    "    ds_val.drop('date', axis = 1, inplace = True)\n",
    "    ds_test.drop('date', axis = 1, inplace = True)\n",
    "    \n",
    "    # X/y split\n",
    "    X_train = ds_train.drop(y_name, axis = 1)\n",
    "    y_train = ds_train[y_name]\n",
    "    \n",
    "    X_val = ds_val.drop(y_name, axis = 1)\n",
    "    y_val = ds_val[y_name]\n",
    "    \n",
    "    X_test = ds_test.drop(y_name, axis = 1)\n",
    "    y_test = ds_test[y_name]\n",
    "    \n",
    "    return X_train, X_val, X_test, y_train, y_val, y_test, cats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, _, X_rev_test, _, _, y_rev_test, _ = prep_data2(data, rev_conts, rev_cats.copy(), rev_name)\n",
    "_, _, X_cos_test, _, _, y_cos_test, _ = prep_data2(data, cos_conts, cos_cats.copy(), cos_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = pd.DataFrame()\n",
    "res['id'] = X_rev_test['id'].copy()\n",
    "X_rev_test.drop('id', axis = 1, inplace = True)\n",
    "X_cos_test.drop('id', axis = 1, inplace = True)\n",
    "\n",
    "rtdata = data[rtmask].copy()\n",
    "bst = train_model(rtdata, rev_conts, rev_cats.copy(), rev_name)\n",
    "res['rt_rev_pred'] = bst.predict(X_rev_test)\n",
    "bst = train_model(rtdata, cos_conts, cos_cats.copy(), cos_name)\n",
    "res['rt_cos_pred'] = bst.predict(X_cos_test)\n",
    "\n",
    "rp = pd.DataFrame()\n",
    "rpdata = data[rpmask].copy()\n",
    "bst = train_model(rpdata, rev_conts, rev_cats.copy(), rev_name)\n",
    "res['rp_rev_pred'] = bst.predict(X_rev_test)\n",
    "bst = train_model(rpdata, cos_conts, cos_cats.copy(), cos_name)\n",
    "res['rp_cos_pred'] = bst.predict(X_cos_test)\n",
    "\n",
    "de = pd.DataFrame()\n",
    "dedata = data[demask].copy()\n",
    "bst = train_model(dedata, rev_conts, rev_cats.copy(), rev_name)\n",
    "res['de_rev_pred'] = bst.predict(X_rev_test)\n",
    "bst = train_model(dedata, cos_conts, cos_cats.copy(), cos_name)\n",
    "res['de_cos_pred'] = bst.predict(X_cos_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res.reset_index().drop('index', axis = 1).set_index('id').to_csv('six_models.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}