Tools/causalml/meta_learners_with_synthetic_data_multiple_treatment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `causalml` - Meta-Learner Example Notebook\n",
    "This notebook only contains regression examples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:47:09.085819Z",
     "start_time": "2020-04-14T18:47:09.066588Z"
    }
   },
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:47:12.227097Z",
     "start_time": "2020-04-14T18:47:09.088487Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jeong/.conda/envs/py36/lib/python3.6/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.utils.testing module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.utils. Anything that cannot be imported from sklearn.utils is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.linear_model import LinearRegression, LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "import statsmodels.api as sm\n",
    "from xgboost import XGBRegressor, XGBClassifier\n",
    "import warnings\n",
    "\n",
    "# from causalml.inference.meta import XGBTLearner, MLPTLearner\n",
    "from causalml.inference.meta import BaseSRegressor, BaseTRegressor, BaseXRegressor, BaseRRegressor\n",
    "from causalml.inference.meta import BaseSClassifier, BaseTClassifier, BaseXClassifier, BaseRClassifier\n",
    "from causalml.inference.meta import LRSRegressor\n",
    "from causalml.match import NearestNeighborMatch, MatchOptimizer, create_table_one\n",
    "from causalml.propensity import ElasticNetPropensityModel\n",
    "from causalml.dataset import *\n",
    "from causalml.metrics import *\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "plt.style.use('fivethirtyeight')\n",
    "pd.set_option('display.float_format', lambda x: '%.4f' % x)\n",
    "\n",
    "# imports from package\n",
    "import logging\n",
    "from sklearn.dummy import DummyRegressor\n",
    "from sklearn.metrics import mean_squared_error as mse\n",
    "from sklearn.metrics import mean_absolute_error as mae\n",
    "import statsmodels.api as sm\n",
    "from copy import deepcopy\n",
    "\n",
    "logger = logging.getLogger('causalml')\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Single Treatment Case"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate synthetic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:47:12.283931Z",
     "start_time": "2020-04-14T18:47:12.230839Z"
    }
   },
   "outputs": [],
   "source": [
    "# Generate synthetic data using mode 1\n",
    "y, X, treatment, tau, b, e = synthetic_data(mode=1, n=10000, p=8, sigma=1.0)\n",
    "\n",
    "treatment = np.array(['treatment_a' if val==1 else 'control' for val in treatment])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## S-Learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:47:13.857975Z",
     "start_time": "2020-04-14T18:47:12.286727Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.6622\n",
      "INFO:causalml:    RMSE (Treatment):     0.6941\n",
      "INFO:causalml:   sMAPE   (Control):     0.6536\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3721\n",
      "INFO:causalml:    Gini   (Control):     0.8248\n",
      "INFO:causalml:    Gini (Treatment):     0.8156\n"
     ]
    }
   ],
   "source": [
    "learner_s = BaseSRegressor(XGBRegressor(), control_name='control')\n",
    "ate_s = learner_s.estimate_ate(X=X, treatment=treatment, y=y, return_ci=False, bootstrap_ci=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:47:13.912096Z",
     "start_time": "2020-04-14T18:47:13.861042Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.57431368])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ate_s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:47:15.541087Z",
     "start_time": "2020-04-14T18:47:13.914579Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.6622\n",
      "INFO:causalml:    RMSE (Treatment):     0.6941\n",
      "INFO:causalml:   sMAPE   (Control):     0.6536\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3721\n",
      "INFO:causalml:    Gini   (Control):     0.8248\n",
      "INFO:causalml:    Gini (Treatment):     0.8156\n"
     ]
    }
   ],
   "source": [
    "alpha = 0.05\n",
    "learner_s = BaseSRegressor(XGBRegressor(), ate_alpha=alpha, control_name='control')\n",
    "ate_s, ate_s_lb, ate_s_ub = learner_s.estimate_ate(X=X, treatment=treatment, y=y, return_ci=True,\n",
    "                                                   bootstrap_ci=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:47:15.593203Z",
     "start_time": "2020-04-14T18:47:15.545759Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.54689052],\n",
       "       [0.57431368],\n",
       "       [0.60173684]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_s_lb, ate_s, ate_s_ub))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Boostrap Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:48:31.923961Z",
     "start_time": "2020-04-14T18:47:15.597096Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.6622\n",
      "INFO:causalml:    RMSE (Treatment):     0.6941\n",
      "INFO:causalml:   sMAPE   (Control):     0.6536\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3721\n",
      "INFO:causalml:    Gini   (Control):     0.8248\n",
      "INFO:causalml:    Gini (Treatment):     0.8156\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [01:14<00:00,  1.34it/s]\n"
     ]
    }
   ],
   "source": [
    "ate_s_b, ate_s_lb_b, ate_s_ub_b = learner_s.estimate_ate(X=X, treatment=treatment, y=y, return_ci=True,\n",
    "                                                         bootstrap_ci=True, n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:48:31.965447Z",
     "start_time": "2020-04-14T18:48:31.926284Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.51141982],\n",
       "       [0.57431368],\n",
       "       [0.64097547]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_s_lb_b, ate_s_b, ate_s_ub_b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:48:33.309900Z",
     "start_time": "2020-04-14T18:48:31.968542Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.6622\n",
      "INFO:causalml:    RMSE (Treatment):     0.6941\n",
      "INFO:causalml:   sMAPE   (Control):     0.6536\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3721\n",
      "INFO:causalml:    Gini   (Control):     0.8248\n",
      "INFO:causalml:    Gini (Treatment):     0.8156\n"
     ]
    }
   ],
   "source": [
    "learner_s = BaseSRegressor(XGBRegressor(), control_name='control')\n",
    "cate_s = learner_s.fit_predict(X=X, treatment=treatment, y=y, return_ci=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:48:33.349476Z",
     "start_time": "2020-04-14T18:48:33.311840Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.37674308],\n",
       "       [0.42519259],\n",
       "       [0.60864675],\n",
       "       ...,\n",
       "       [0.19940662],\n",
       "       [0.35013032],\n",
       "       [0.78372002]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:49:37.587994Z",
     "start_time": "2020-04-14T18:48:33.351595Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.6622\n",
      "INFO:causalml:    RMSE (Treatment):     0.6941\n",
      "INFO:causalml:   sMAPE   (Control):     0.6536\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3721\n",
      "INFO:causalml:    Gini   (Control):     0.8248\n",
      "INFO:causalml:    Gini (Treatment):     0.8156\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [01:02<00:00,  1.59it/s]\n"
     ]
    }
   ],
   "source": [
    "alpha = 0.05\n",
    "learner_s = BaseSRegressor(XGBRegressor(), ate_alpha=alpha, control_name='control')\n",
    "cate_s, cate_s_lb, cate_s_ub = learner_s.fit_predict(X=X, treatment=treatment, y=y, return_ci=True,\n",
    "                               n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:49:37.669038Z",
     "start_time": "2020-04-14T18:49:37.591481Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.37674308],\n",
       "       [0.42519259],\n",
       "       [0.60864675],\n",
       "       ...,\n",
       "       [0.19940662],\n",
       "       [0.35013032],\n",
       "       [0.78372002]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:49:37.759221Z",
     "start_time": "2020-04-14T18:49:37.674451Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.18972662],\n",
       "       [ 0.20548496],\n",
       "       [ 0.09983036],\n",
       "       ...,\n",
       "       [-0.62837307],\n",
       "       [-0.19766161],\n",
       "       [-0.07736247]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_s_lb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:49:37.856614Z",
     "start_time": "2020-04-14T18:49:37.764939Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.8139405 ],\n",
       "       [1.278447  ],\n",
       "       [1.21720439],\n",
       "       ...,\n",
       "       [0.90244564],\n",
       "       [0.9450083 ],\n",
       "       [1.1529291 ]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_s_ub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## T-Learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:49:39.262164Z",
     "start_time": "2020-04-14T18:49:37.860129Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n"
     ]
    }
   ],
   "source": [
    "learner_t = BaseTRegressor(XGBRegressor(), control_name='control')\n",
    "ate_t, ate_t_lb, ate_t_ub = learner_t.estimate_ate(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:49:39.304936Z",
     "start_time": "2020-04-14T18:49:39.264017Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.55534845],\n",
       "       [0.58090983],\n",
       "       [0.60647121]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_t_lb, ate_t, ate_t_ub))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Boostrap Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:50:40.797544Z",
     "start_time": "2020-04-14T18:49:39.307236Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [01:00<00:00,  1.66it/s]\n"
     ]
    }
   ],
   "source": [
    "ate_t_b, ate_t_lb_b, ate_t_ub_b = learner_t.estimate_ate(X=X, treatment=treatment, y=y, bootstrap_ci=True,\n",
    "                                                   n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:50:40.836006Z",
     "start_time": "2020-04-14T18:50:40.799256Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.51343277],\n",
       "       [0.58090983],\n",
       "       [0.65843097]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_t_lb_b, ate_t_b, ate_t_ub_b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:50:41.916753Z",
     "start_time": "2020-04-14T18:50:40.837869Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n"
     ]
    }
   ],
   "source": [
    "learner_t = BaseTRegressor(XGBRegressor(), control_name='control')\n",
    "cate_t = learner_t.fit_predict(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:50:41.956040Z",
     "start_time": "2020-04-14T18:50:41.918664Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.23669004],\n",
       "       [-0.0793891 ],\n",
       "       [-0.10774326],\n",
       "       ...,\n",
       "       [ 0.30539629],\n",
       "       [ 0.50784194],\n",
       "       [ 0.00356007]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:51:42.554430Z",
     "start_time": "2020-04-14T18:50:41.963277Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [00:59<00:00,  1.68it/s]\n"
     ]
    }
   ],
   "source": [
    "learner_t = BaseTRegressor(XGBRegressor(), control_name='control')\n",
    "cate_t, cate_t_lb, cate_t_ub = learner_t.fit_predict(X=X, treatment=treatment, y=y, return_ci=True, n_bootstraps=100,\n",
    "                                                    bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:51:42.599599Z",
     "start_time": "2020-04-14T18:51:42.559391Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.23669004],\n",
       "       [-0.0793891 ],\n",
       "       [-0.10774326],\n",
       "       ...,\n",
       "       [ 0.30539629],\n",
       "       [ 0.50784194],\n",
       "       [ 0.00356007]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:51:42.639342Z",
     "start_time": "2020-04-14T18:51:42.601624Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.6752711 ],\n",
       "       [-0.72038152],\n",
       "       [-1.2330182 ],\n",
       "       ...,\n",
       "       [-0.82131582],\n",
       "       [-0.48846376],\n",
       "       [-0.39046848]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_t_lb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:51:42.678368Z",
     "start_time": "2020-04-14T18:51:42.641296Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.66480025],\n",
       "       [1.60697527],\n",
       "       [2.06829221],\n",
       "       ...,\n",
       "       [1.64941401],\n",
       "       [1.59083122],\n",
       "       [1.53139764]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_t_ub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": false
   },
   "source": [
    "## X-Learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:51:44.935095Z",
     "start_time": "2020-04-14T18:51:42.680407Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n"
     ]
    }
   ],
   "source": [
    "learner_x = BaseXRegressor(XGBRegressor(), control_name='control')\n",
    "ate_x, ate_x_lb, ate_x_ub = learner_x.estimate_ate(X=X, treatment=treatment, y=y, p=e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:51:44.972119Z",
     "start_time": "2020-04-14T18:51:44.936710Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.51454586],\n",
       "       [0.53721713],\n",
       "       [0.55988839]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_x_lb, ate_x, ate_x_ub))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:51:47.712668Z",
     "start_time": "2020-04-14T18:51:44.974067Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n"
     ]
    }
   ],
   "source": [
    "ate_x_no_p, ate_x_lb_no_p, ate_x_ub_no_p = learner_x.estimate_ate(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:51:47.750441Z",
     "start_time": "2020-04-14T18:51:47.714685Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.51334384],\n",
       "       [0.53600211],\n",
       "       [0.55866038]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_x_lb_no_p, ate_x_no_p, ate_x_ub_no_p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:51:47.793093Z",
     "start_time": "2020-04-14T18:51:47.752418Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'treatment_a': {'all training': LogisticRegressionCV(Cs=array([1.00230524, 2.15608891, 4.63802765, 9.97700064]),\n",
       "                       class_weight=None,\n",
       "                       cv=StratifiedKFold(n_splits=3, random_state=None, shuffle=True),\n",
       "                       dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "                       l1_ratios=array([0.001     , 0.33366667, 0.66633333, 0.999     ]),\n",
       "                       max_iter=100, multi_class='auto', n_jobs=None,\n",
       "                       penalty='elasticnet', random_state=None, refit=True,\n",
       "                       scoring=None, solver='saga', tol=0.0001, verbose=0)}}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner_x.propensity_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Boostrap Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:53:45.532120Z",
     "start_time": "2020-04-14T18:51:47.795412Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [01:55<00:00,  1.15s/it]\n"
     ]
    }
   ],
   "source": [
    "ate_x_b, ate_x_lb_b, ate_x_ub_b = learner_x.estimate_ate(X=X, treatment=treatment, y=y, p=e, bootstrap_ci=True,\n",
    "                                                   n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:53:45.570961Z",
     "start_time": "2020-04-14T18:53:45.534229Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.46262759],\n",
       "       [0.53721713],\n",
       "       [0.59662513]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_x_lb_b, ate_x_b, ate_x_ub_b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:55:44.972969Z",
     "start_time": "2020-04-14T18:53:45.572878Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [01:56<00:00,  1.17s/it]\n"
     ]
    }
   ],
   "source": [
    "ate_x_b_no_p, ate_x_lb_b_no_p, ate_x_ub_b_no_p = learner_x.estimate_ate(X=X, treatment=treatment, y=y, bootstrap_ci=True,\n",
    "                                                   n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:55:45.012081Z",
     "start_time": "2020-04-14T18:55:44.975086Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.44360865],\n",
       "       [0.53598752],\n",
       "       [0.59794413]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_x_lb_b_no_p, ate_x_b_no_p, ate_x_ub_b_no_p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:55:47.269808Z",
     "start_time": "2020-04-14T18:55:45.013958Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n"
     ]
    }
   ],
   "source": [
    "learner_x = BaseXRegressor(XGBRegressor(), control_name='control')\n",
    "cate_x = learner_x.fit_predict(X=X, treatment=treatment, y=y, p=e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:55:47.308060Z",
     "start_time": "2020-04-14T18:55:47.271872Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.05178452],\n",
       "       [0.01907274],\n",
       "       [0.79584839],\n",
       "       ...,\n",
       "       [0.18147876],\n",
       "       [0.34742898],\n",
       "       [0.23145415]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:55:50.057658Z",
     "start_time": "2020-04-14T18:55:47.310097Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n"
     ]
    }
   ],
   "source": [
    "cate_x_no_p = learner_x.fit_predict(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:55:50.095258Z",
     "start_time": "2020-04-14T18:55:50.059363Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.06426511],\n",
       "       [0.0189166 ],\n",
       "       [0.78233515],\n",
       "       ...,\n",
       "       [0.2237187 ],\n",
       "       [0.29647103],\n",
       "       [0.2359861 ]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_no_p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:57:07.153422Z",
     "start_time": "2020-04-14T18:55:50.097185Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [01:14<00:00,  1.34it/s]\n"
     ]
    }
   ],
   "source": [
    "learner_x = BaseXRegressor(XGBRegressor(), control_name='control')\n",
    "cate_x, cate_x_lb, cate_x_ub = learner_x.fit_predict(X=X, treatment=treatment, y=y, p=e, return_ci=True,\n",
    "                                                     n_bootstraps=100, bootstrap_size=3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:57:07.202131Z",
     "start_time": "2020-04-14T18:57:07.155610Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.05178452],\n",
       "       [0.01907274],\n",
       "       [0.79584839],\n",
       "       ...,\n",
       "       [0.18147876],\n",
       "       [0.34742898],\n",
       "       [0.23145415]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:57:07.252726Z",
     "start_time": "2020-04-14T18:57:07.205064Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.71763188],\n",
       "       [-0.79487709],\n",
       "       [-0.329782  ],\n",
       "       ...,\n",
       "       [-0.57672694],\n",
       "       [-0.48450804],\n",
       "       [-0.43157597]])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_lb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:57:07.298204Z",
     "start_time": "2020-04-14T18:57:07.254908Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.40320321],\n",
       "       [1.59906792],\n",
       "       [1.59324502],\n",
       "       ...,\n",
       "       [1.07747513],\n",
       "       [1.30836353],\n",
       "       [1.18985624]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_ub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:58:26.822473Z",
     "start_time": "2020-04-14T18:57:07.300843Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4868\n",
      "INFO:causalml:    RMSE (Treatment):     0.5434\n",
      "INFO:causalml:   sMAPE   (Control):     0.5230\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3114\n",
      "INFO:causalml:    Gini   (Control):     0.9216\n",
      "INFO:causalml:    Gini (Treatment):     0.8988\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [01:16<00:00,  1.31it/s]\n"
     ]
    }
   ],
   "source": [
    "cate_x_no_p, cate_x_lb_no_p, cate_x_ub_no_p = learner_x.fit_predict(X=X, treatment=treatment, y=y, return_ci=True,\n",
    "                                                     n_bootstraps=100, bootstrap_size=3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:58:26.864389Z",
     "start_time": "2020-04-14T18:58:26.824577Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.06430496],\n",
       "       [0.01891659],\n",
       "       [0.78209735],\n",
       "       ...,\n",
       "       [0.22376976],\n",
       "       [0.29645377],\n",
       "       [0.23597794]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_no_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:58:26.906146Z",
     "start_time": "2020-04-14T18:58:26.866620Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.62013372],\n",
       "       [-0.90236405],\n",
       "       [-0.31043938],\n",
       "       ...,\n",
       "       [-0.54219561],\n",
       "       [-0.2852425 ],\n",
       "       [-0.37437315]])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_lb_no_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:58:26.945545Z",
     "start_time": "2020-04-14T18:58:26.908137Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.4199368 ],\n",
       "       [1.45096372],\n",
       "       [1.57656827],\n",
       "       ...,\n",
       "       [1.34583137],\n",
       "       [1.37899369],\n",
       "       [1.25074382]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_ub_no_p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## R-Learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:58:33.047158Z",
     "start_time": "2020-04-14T18:58:26.947521Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "ate_r, ate_r_lb, ate_r_ub = learner_r.estimate_ate(X=X, treatment=treatment, y=y, p=e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:58:33.087239Z",
     "start_time": "2020-04-14T18:58:33.049284Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.55904178],\n",
       "       [0.55951123],\n",
       "       [0.55998069]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_r_lb, ate_r, ate_r_ub))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:58:38.036497Z",
     "start_time": "2020-04-14T18:58:33.089093Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n"
     ]
    }
   ],
   "source": [
    "ate_r_no_p, ate_r_lb_no_p, ate_r_ub_no_p = learner_r.estimate_ate(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:58:38.072770Z",
     "start_time": "2020-04-14T18:58:38.038825Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.49307912],\n",
       "       [0.49354918],\n",
       "       [0.49401924]])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_r_lb_no_p, ate_r_no_p, ate_r_ub_no_p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T18:58:38.109243Z",
     "start_time": "2020-04-14T18:58:38.074501Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'treatment_a': {'all training': LogisticRegressionCV(Cs=array([1.00230524, 2.15608891, 4.63802765, 9.97700064]),\n",
       "                       class_weight=None,\n",
       "                       cv=KFold(n_splits=5, random_state=None, shuffle=True),\n",
       "                       dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "                       l1_ratios=array([0.001     , 0.33366667, 0.66633333, 0.999     ]),\n",
       "                       max_iter=100, multi_class='auto', n_jobs=None,\n",
       "                       penalty='elasticnet', random_state=None, refit=True,\n",
       "                       scoring=None, solver='saga', tol=0.0001, verbose=0)}}"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner_r.propensity_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Boostrap Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:00:38.045754Z",
     "start_time": "2020-04-14T18:58:38.111041Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [01:56<00:00,  1.17s/it]\n"
     ]
    }
   ],
   "source": [
    "ate_r_b, ate_r_lb_b, ate_r_ub_b = learner_r.estimate_ate(X=X, treatment=treatment, y=y, p=e, bootstrap_ci=True,\n",
    "                                                   n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:00:38.089296Z",
     "start_time": "2020-04-14T19:00:38.047834Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.37951505],\n",
       "       [0.54612646],\n",
       "       [0.53701368]])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_r_lb_b, ate_r_b, ate_r_ub_b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:03:24.546297Z",
     "start_time": "2020-04-14T19:00:38.091485Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [02:42<00:00,  1.63s/it]\n"
     ]
    }
   ],
   "source": [
    "ate_r_b_no_p, ate_r_lb_b_no_p, ate_r_ub_b_no_p = learner_r.estimate_ate(X=X, treatment=treatment, y=y, bootstrap_ci=True,\n",
    "                                                   n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:03:24.617403Z",
     "start_time": "2020-04-14T19:03:24.549832Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.37126915],\n",
       "       [0.50635052],\n",
       "       [0.51400059]])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_r_lb_b_no_p, ate_r_b_no_p, ate_r_ub_b_no_p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:03:29.033458Z",
     "start_time": "2020-04-14T19:03:24.621209Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "cate_r = learner_r.fit_predict(X=X, treatment=treatment, y=y, p=e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:03:29.087607Z",
     "start_time": "2020-04-14T19:03:29.036023Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.57365084],\n",
       "       [-0.63619554],\n",
       "       [-0.05320793],\n",
       "       ...,\n",
       "       [ 0.56346375],\n",
       "       [ 0.56288183],\n",
       "       [ 0.87085617]])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:03:33.641108Z",
     "start_time": "2020-04-14T19:03:29.090259Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n"
     ]
    }
   ],
   "source": [
    "cate_r_no_p = learner_r.fit_predict(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:03:33.692456Z",
     "start_time": "2020-04-14T19:03:33.644376Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.19582933],\n",
       "       [-0.29006499],\n",
       "       [ 0.46513131],\n",
       "       ...,\n",
       "       [ 0.89712083],\n",
       "       [ 0.81002617],\n",
       "       [ 0.82598114]])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r_no_p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:23.515500Z",
     "start_time": "2020-04-14T19:03:33.694879Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [00:46<00:00,  2.15it/s]\n"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "cate_r, cate_r_lb, cate_r_ub = learner_r.fit_predict(X=X, treatment=treatment, y=y, p=e, return_ci=True,\n",
    "                                                     n_bootstraps=100, bootstrap_size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:23.561881Z",
     "start_time": "2020-04-14T19:04:23.517576Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.43967736],\n",
       "       [-0.27467608],\n",
       "       [-0.36704457],\n",
       "       ...,\n",
       "       [ 1.70213294],\n",
       "       [ 0.53581667],\n",
       "       [ 0.67119908]])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:23.608087Z",
     "start_time": "2020-04-14T19:04:23.564124Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-2.36270347],\n",
       "       [-2.10110987],\n",
       "       [-3.33190218],\n",
       "       ...,\n",
       "       [-2.25005704],\n",
       "       [-2.08611215],\n",
       "       [-1.89283199]])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r_lb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:23.655535Z",
     "start_time": "2020-04-14T19:04:23.610212Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.23361461],\n",
       "       [4.39421365],\n",
       "       [3.95620847],\n",
       "       ...,\n",
       "       [3.15905744],\n",
       "       [3.23586204],\n",
       "       [2.31788745]])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r_ub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:58.689399Z",
     "start_time": "2020-04-14T19:04:23.658096Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [00:31<00:00,  3.14it/s]\n"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "cate_r_no_p, cate_r_lb_no_p, cate_r_ub_no_p = learner_r.fit_predict(X=X, treatment=treatment, y=y, return_ci=True,\n",
    "                                                     n_bootstraps=100, bootstrap_size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:58.736814Z",
     "start_time": "2020-04-14T19:04:58.691749Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.14972556],\n",
       "       [ 0.18446118],\n",
       "       [ 0.23380044],\n",
       "       ...,\n",
       "       [ 0.55917108],\n",
       "       [-0.16540062],\n",
       "       [ 0.62050438]])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r_no_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:58.783181Z",
     "start_time": "2020-04-14T19:04:58.739229Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-2.37674593],\n",
       "       [-1.66803797],\n",
       "       [-3.47868801],\n",
       "       ...,\n",
       "       [-1.95877534],\n",
       "       [-2.32770172],\n",
       "       [-1.68704787]])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r_lb_no_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:58.843766Z",
     "start_time": "2020-04-14T19:04:58.798145Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2.9130644 ],\n",
       "       [3.99895564],\n",
       "       [3.61212277],\n",
       "       ...,\n",
       "       [3.174209  ],\n",
       "       [3.38644627],\n",
       "       [2.62858756]])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r_ub_no_p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:59.305207Z",
     "start_time": "2020-04-14T19:04:58.849620Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAyIAAAHwCAYAAABaJNtTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzde3wU9b3/8dc34RIETLio8ENFi9gAcoiIgiGRcBEIIFSoB0EUtFABjw3VIqBVIhRzEKwgolY8LaCBciS0XCQgiMFEBKrHICBoBATRoCImIUDIbX5/zGabbHY3GzYXwr6fj8c+yMxn5jufmZ2E+ex3vrPGsixERERERERqUlBtJyAiIiIiIoFHhYiIiIiIiNQ4FSIiIiIiIlLjVIiIiIiIiEiNUyEiIiIiIiI1ToWIiIiIiIjUOBUiIoIxZqkxZms1tT3OGFPoaboathdvjPmqutqvLGNMZ2PMbmNMnjHm69rOp64xxvzJGHPQ07Qf7R43xkz3t52qYIx53hjzgzHGMsaMqe18RERqigoRkUuUo7iwHK8CY8xJY0yaMeYJY0xjl8XjgHsq0XahMWacj4uvAtr42nYlcohy7Nt1LqH5QI+q3p4fngdygHDgVk8LGWPqGWMedRQtp40xOcaYT40xTxljmrlZ/nNjTJExppPL/OOl3nd3r0LHcm95iGdVkGPpZXONMemVOBeqwn8DUb4u7KXIvhlYVGVZXSBjTE9gKvAQ0BpYXcXtV2vh72GbMY7z4+qa3G5FjDEpxpg3ajsPEfk3FSIil7ZU7IubtkBvIBH4L+D/jDFXlSxkWVa2ZVk/V+WGja2+ZVnnLMv6virb9sayrFzLsk7W1PZ80B7YblnW15Zl/ehuAWNMfeAdYA7wv0Af4D+Ap7CLqrEuy98BNAf+B/itS3M3Y7/nrYHbHfMGl5pXuih8v9T8kteNPuzTRMeyNwNbgL8ZY+72sG9BxphgH9r0SVW9v5Zl/WhZ1pmqyMlP7YF8y7I2WJZ1wrKsvAtpxBjTwJ8k/F1fROSCWJall156XYIvYCmw1c38NsAp4G+elgU6AZuBLOAMcAC43xH7GrBKvxzzxwGF2AXPp0A+EFsyv1TbJcv1A/YDecAuIMJ1GZe8r3ZsLwa4zjUHIMWxXDzwlcu6Y4HPHTkdB/4E1CsVTwHeAJ4GTjiOz3KgSQXHuDXwd8dxOudop5sj5i7HeA/tPA4UA7d7iDdzmX4LeAHo7sg1xMN6Nzi228NN7C1gUyXPqXqO9u51mX8EeNPx85+Ag8Ao4AvHe93eEbsP2ON4z49g915dVqqdRsDrQLZjvxZj9ygdLLXMn0pPO+b1B9KAs473IgW43rGs63swxrHOcWB6qTZCgSXAj478dgN93RzLX2MXjWeBQzh+L0ot97Bj//OAnxy5/D8Px/Mtl9wKHfMNMM1xjPId23nUZd3jwLPAa47tfOim/X5u9v8NRyzNcayfAzKB44759YHZ2L/n54B9wHiXdn/veB9zHeuuAFq5HKfSr62lzzlgiiP/XOAv2OfVI8Ax4GfgVaB+qe0Z7F7bLxzH9UtgBmV/h48Dz2D3cv0MfI99fgV7ONYWEOXDOT/GcS7kACeBDcANlfm90UsvvTy/1CMiEmAsy/oWu2dkuDHG09+AldgXN5FAZ+Ax7P/cwb69qAj7YqLkU/QSQcBcx/LhwMce2g/CvsCcDNyGffH3jjGmkY+78Q0wzPHzbY4chrtb0BgzGPgr8CZwE/ZF/yPATJdFf43dyxAD3AsMwb4YdMsYY4B/Yu/nEEce3wNbjDEtHTm2xr5Amuv4eb6H5u4HtlmW9ZG7oFWqt8oY09yR61LLsnY5tvmfnvKsIeewL2BLXIPdU3M/dlGbaYwZD7wEzAM6YhebA7GLjRLPY7+v92Ofe/nYvS8eGWMGAMnYF4u3O16J2Be3/43dw1TSM+jt1qelQF/sYulmR3sbjTHtXZabC/wNu8dqNXZvUDtHLt2Bl7Ev5H+JfS6t8JL+I9jn43nK9lb9Dvv8nIN9/F4A5htjxrqs/3vgW+xes/Fu2v8A+/e0iH/v/2Ol4qOwC7A+2B8a4Ni3uxztdcQu5l5ws+3HsP82jAB+gX3MwS6eRjh+7urYZunbPm/HPnZ3Yl/kPwisB7oBA4AHHPPGlVpntmM/ngA6OPb7EeweQ9fjcQz7dzHOsU7JmJtHgI+w34+SY7GLijXELvhuxi54DbDB0YspIv6q7UpIL730qp4XHnpEHLGJ2J8IXuluWexPpMd5abvQNY594WAB0W7mu/aIWJT9tLkZ9qejv3G3jmOes0fEMR3lmL7OZbl4SvWIYF+E/q/LMnHYF88NHNMpwB6XZV4FPvJyDPo6tt+x1LyG2J8QP1Nq3tfAHyt4r84CL/n4vv4e+KTU9HQgzcOyFfWIFDqOe+nXP7xsu0yPCHbx8bBj3njHvD9hX/i2cVn3OOU/We+D3RPUFLgc+4L8QZdlPsVLjwj2xeU/K/t7QKkeEeyiwQL6l4ob7E/9X3c5lr9zOR5nS52392AX7E0r8Xs6HshzmZcJPOcybxHwpUv+m31ofxwuv0uO+WnYPZ2m1Lz2jn28wWXZWcDHXrZxq2O9qxzTMY7pq92cc5mU7e3YjF1MNyg17x3g746fm2D/rvZzaesh4KTL8VjjsswWHD11jukUHD1CF/oCrnDsW3d/2tFLL73sl3pERAKTcfxreYjPB95wDO6MN8Z0rUTb//JxOeen/5b9if8B7E9/q1on7E+GS9sOhADtSs3b47LMd8BVeNYJ+MmyrM9LZliWdR77U9bK7oepeBGnCdgX1yXeAm53HbTuox1AhMtrsg/rLTXG5GJfIM7D/uT+f0rFv7PsnjcAjDEln/a/5BjgnutYfz32vt/geDVw5FTahxXk0hV414ecvSk5dqklMyzLshzTrsc1vdQyhdi9eSXnyWbsT+SPGGNWGmMmGGNaVCYRR49XK9yfs+2MMQ1Lzdtdmbbd+NixnyW6Of5Nd3mfnsAuUkpy7GOMedcY840x5jT2BT7YY9Eq8rllWQWlpk9gF5b5LvOudPzcGft3da1LTouBFi4PckinrIp+hytkjOlqjPmnMeZrx74ecYR82VcRqUC92k5ARGpFJ+xej5/cBS3Lmm2MScS+daYP8KQx5nnLsv5YQbtF1gUOtnVR7GZedd8Kke8ybVFzD/T4Avs2GK8cg9Q7AC8aY14sFQrCvhUqrpLbPWtZ1oU86nga9qfWucD3LhezYI8rKq3kOP4X5S+wwb6N7UKL0MoUcVXB43liWVaOMeYW7N66vti3Az1vjOltWZbrRXJV8Hewvaf3qQf2WIzSigGMMddjv/dLsXsff8K+KN+MXUhWpMBl2vIwrySXkn/vBg67aS+n1M9V+jtsjGmKXeS+j92zdAIIxh43o8H9IlVAPSIiAcYY0wb7Pvg1lmW5u+AHwLKsw5ZlvWJZ1q+xB4FOKhXOx/4P2R/OR+waY8KwL7BLehd+AIJLP9kL+5Pv0kouOirKYz9wh8u8Xtif5h+qTMJu2m1hjHEWEI5Pq7tjX6hUxltAH2PM7e6CpT71/S327SauvRiPAfcbY0Iqud0L9b1lWV9Z9lOePPWqlfYd9i05NzrWc32dB77CviCNdFnXddrVJ9j37nviy7m63/FvdMkMxxigaCr5XlqWVWhZVoplWU9jjyv4EXsshq/rn8K+4HV3zpYcq8rIB4Ic+1ORTxz/Xu3mPSopAm7DvgUxzrKsHZZlfYHdg+O6TfD/bwTAXuxb9n7h4dwpqkRblf271RFoATzpeE8POqZFpIqoR0Tk0tbAGNMK+0OHFtif1M7AvtCf4W4FY0wT7AG5Sdi3IYRh94x8XmqxI0BvY0wy9qNHK/s4VQv7k+KSQfBzgNP8e2Dvbsf0fxtjnsO+heoZlzaOYn9KO8gYswo4b1lWtpttJQDrjf3ldWuwL9zjgRdcbgeprG2OPFcYYx7B7mF6Gvs2klcr2dZC7IG6m40xs7BvdfkRuzibCLxvjHkTe5D6BMuyylwcG2OOYu/nf2I/7ctXJeeHK3e9HBfMsizLGPMU8JoxJhtYhz0+pSP2uIxJjt6EJcBzxpgfgQzswusG7ELGk1nYDzr4M/an9OeBnkCqZVkZ2OfqMEfB+ANw2vVi3rKsL4wx/3Dk9zD2eINHsB9E8Gtf99MYMxx7oH4q9hOWbsW+Je1zb+u5kYB97h/C7kHqh30sXB/V7Isj2L1GQ4wxO4FzlmXlulvQsqyDxpjlwF+NMVOBndhjNLoBzS3Lmof9xCoDPG6M+Tv275NrT+lR7N/xwcaY1dhjYHK4AI7zYi4w11FMvYfdO9oF6GxZltu/Yx4cASIdDxfIBrIct9d58jV28fI7Rw9kO+ynjIlIFVGPiMilLRr7k+hj2Be392E/1aer5fm7PQqxB4//D/a4jZLBpKNLLfM4cAv2f9RuvxujAsXAk9iP7vwY+xPVwZZlnQXnp8KjsHtNPsO+wH+idAOO/GdgD9bOBNa625BlWRuxB7aOxf50+0XgFewn4Vwwx4X6r7Af1foO9tiYVsCdlS3MHPfMx2Lv573Y4wH2Yl+Q7gaW8e/vEim3n5ZlncZ+clRlL1R7Yx8711doJdupkGVZf8N+T4dhv+f/wi4uvy212FTsY7kCe6xNY+zH03prNxn7qWWR2MdqF/aTkkpu91mCPeB9J/a56umLOx/EvshdiT3W4DZgkKOY8dUp7HPiXewL9gTgWcuyllWiDbAHpj+LfYG/H/gDMPUC2sGyn8T2MvaT434AFlSwym8c238G+/d/K/ZTzA472vsU+xbAR7ALrN9jP52q9Da/xX6i1R+xz6c1lc3bpb2Z2OfGROzfizTsJ4t9Xcmm5mEXIHuwzwWvX3zq+BtzP/bv5ufYH9D8vpLbFBEvTBV+6CUiIiIiIuIT9YiIiIiIiEiNUyEiIiIiAccY83TpRwK7vLJqOz+RQKBbs0RERCTgOL6zpbmHsGVZlj9P1RMRH9SpQiQ7O7vuJCsiIiIiIgCEhoaWe4y4bs0SEREREZEap0JERERERERqnAqRS1xGRmUegS+XKp0HonNAQOeB6BwQ28VyHqgQERERERGRGqdCREREREREaly92k5ARERERKqHZVnk5uZSXFwMQEhICNnZ2bWcldS26jgPgoKCaNKkCcaUeziWRypERERERC5Rubm5NGzYkAYNGgDQsGFDQkJCajkrqW3VcR7k5+eTm5tL06ZNfV5Ht2aJiIiIXKKKi4udRYhIdWrQoIGz581XKkRERERERKTG6dYsERERkQAQ9rdvq6XdrAfbVEu7culTj4iIiIiIiNQ49YiIiIiIBJCq6sHwpYfl1KlTDB06FIAffviB4OBgWrRoAcC2bduc41c6d+5MSkqKM3ax2rNnD0uWLOHll18mNTWVBg0a0L179ypp++jRo+zevZt77rmnStrztp0PP/yQ0aNHV3rdkydP8vDDD5OUlFQluahHRERERESqRfPmzUlLSyMtLY0HH3yQyZMnO6ercxB9YWFhtbTx5z//mYcffhiAtLQ0du/eXWXbP3bsGKtXr670eheynTVr1lzQui1btuSqq65i586dVZKLekRERERE5KJz5swZnnjiCQ4cOEBBQQHTp09n8ODBHD16lIkTJ3LmzBkA5s2bR/fu3UlNTeW5554jNDSUjIwM1qxZwz333EOPHj3YvXs3rVu3ZsWKFTRq1IgjR47whz/8gZMnT3LZZZexcOFCbrzxRiZNmkRISAifffYZ3bt357nnnnPmc/r0afbv30/nzp05evQof/vb3wgODmbVqlU8//zzvPnmm2XWfeqppyqV/7PPPsuXX35JVFQUo0aNIiwsjHfeeYezZ89y6NAhHn30UfLz81m1ahUNGzbk7bffplmzZl73pWnTpqSnp/P9998za9Yshg0bxrPPPssXX3zh3M4jjzxS7th7yhFg8ODBvP322/To0cPv91iFiIiIiIhcdF544QXuuOMOFi9eTFZWFn379iUmJoYrrriCf/zjH4SEhHDo0CF+85vfkJKSAti3Tu3YsYPrrruOo0ePcujQId544w1eeuklxo0bx7p16xg5ciRxcXG8+OKLtGvXjo8//pjHH3+c9evXA/Ddd9/x7rvvEhwcXCafTz/9lA4dOgDQtm1bHnzwQZo0acKjjz4KwJtvvllm3VmzZlUq/5kzZ/Lyyy+zatUqABITEzlw4AAffPAB58+fp2vXrsTHx5OamsqMGTNYuXIlkydP9rov33//PZs2beLLL79k1KhRDBs2jJkzZ7Jw4UKvvS/ejvHNN9/MnDlzquQ9ViEiIiIiIhedbdu2kZyczKJFiwA4f/48x48fp1WrVkydOpV9+/YRFBTEoUOHnOt07dqV6667zjndtm1b/uM//gOAiIgIjh07Rm5uLrt372bs2LHO5fLz850/Dxs2rFwRAvZFfcuWLb3mXHrdC8nfVXR0NE2bNqVp06ZcfvnlDBw4EICOHTuyf//+Cvdl8ODBBAUFER4ezo8//ug199IKCgo85njFFVeQmZnpc1veqBARERERCSDV9RjfqmZZFsuXL6d9+/Zl5ickJHDllVeSlpZGcXExV111lTPWuHHjMss2bNjQ+XNwcDDnzp2juLiY0NBQ0tLS3G7XtY0SISEh5OXlec259LoXkr+r0vkbY5zTQUFBFBUVVbgvpde3LMtr7qW98sorHnPMy8ujUaNGPrfljQari4iIiMhFp2/fvrz++uvOC+g9e/YAkJOTw1VXXUVQUBB///vfKSoqqlS7l19+OW3btuWf//wnYF+g7927t8L1fvnLX3LkyBHndJMmTTh9+nSV5d+0aVOv7VXVvjRt2pTc3Fyvy3g7xocOHXLeouYv9YiIiIiIBICsB9uQl5dHSEhIbafiVs+ePQkKsj8j/9WvfsXTTz/NjBkz6NmzJ8XFxbRt25ZVq1Yxfvx47r//fv7+97/Tr18/jz0Y3rz++us8/vjjzJs3j8LCQoYPH07nzp29rnPjjTeSk5PD6dOnadq0KbGxsTzwwANs3LiR559/vtzyU6dOrVT+nTp1Ijg4mJ49ezJ69GjCwsKqZV9ct+NusLq3Y5yamkr//v19yq0ipjLdNLUtOzu77iR7kcjIyCjXJSiBR+eB6BwQ0HkQiLKzswkNDXVOX8yFSF2wePFimjZtygMPPFDbqfjFn/MgNjaWlStXui2UXM+30kJDQ43rPN2aJSIiIiLig9/85jfV+v0nF7uTJ0/yyCOP+NxbUxHdmiUiItUubmscAAv7LazlTERELlxISAj33ntvbadRZd577z1mzpxZZl7btm1JTEx0u3zLli0ZMmRIlW1fhYiIiFQ7FSAiIhefvn370rdv31rbvm7NEhERERGRGqdCREREql1mbiaZuVXzBVgiInJp0K1ZIiJS7Tq8YT9zPmtKVi1nIhK4moyNoUk1tJu7LKUaWpVAoEJERESqXavGrWo7BRERucioEBERkWp3cMLB2k5BRByqqgejydiYCpc5fvw4gwYNYvv27TRr1oysrCzuuOMO1q9fT9u2bcss26ZNG7799tsqya06bdiwgf379zNt2jQ2bNjADTfcQHh4eJW0/dlnn3HixIkq+8JAT/bt28epU6cuaDv79+/n5Zdf5tVXX/U7D40REREREZFqcfXVV/Ob3/yG+Ph4AOLj4xk3bly5IqQ6FBYWVksbL730EuPHjwfgnXfe4Ysvvqiy7e/du5ctW7ZUer3K2rdv3wVvp1OnTnz33Xd88803fuehQkREREREqs3kyZP517/+xSuvvMLOnTt59NFHfV735MmT3H///fTu3ZvevXuzc+dOAD755BPuvPNOoqOj6d+/PxkZGQAkJiZy7733ctdddzF06FBSU1MZPHgwDzzwALfeeisTJkzAsiwA0tPTGTRoEL169WL48OGcOHECgMGDBzN9+nRiYmLKfer/1Vdf0aBBA1q0aMGuXbtITk7m6aefJioqiiNHjpRbtzL55+fnk5CQwJo1a4iKimLNmjUkJCQwceJEYmNjuemmm1i3bh3PPPMMkZGRjBgxgoKCggr3ZebMmfTp04dbbrmFHTt2kJ+fz7x588psxx1Pxxhg4MCBHterDN2aJSIi1a7Xil4AbB+9vZYzEZGaVr9+fWbPns2IESP4xz/+Qf369X1ed/r06UyePJnbb7+db775hhEjRrB7927at29PcnIy9erVIyUlhVmzZvHmm28C9u1NH374Ic2aNSM1NZW9e/fy0Ucf0bp1awYMGMDOnTvp1q0bTzzxBCtWrKBly5asWbOG2bNns3jxYgAKCgpISUkpl8/OnTvp0qULAN27dyc2NpaBAwcybNgw5zKl1x0/fnyl8p8xYwbp6enMmzcPgISEBL7++mvWr1/PwYMH6d+/P8uXL2fWrFncd999bN68mQEDBnjdl8LCQrZt28a7777L3LlzWbt2LVOnTmX//v3O7bjj7RjffPPNvPjii8TFxfn8XrqjQkRERKrdnh/21HYKIlKLtmzZQqtWrfj888/p3bu3z+ulpKRw8OC/x5idPn2a3NxccnJymDRpEocPH8YY4+wZAIiJiaFZs2bO6a5du9KmTRsAOnfuzLFjxwgNDeXAgQP86le/AqC4uJirrrrKuc7dd9/tNp/vv/+eli1bes259LoXkr+rfv36Ub9+fTp16kRRURH9+vUDoGPHjhw7doyMjAyv+3LXXXcBEBERwbFjx7zmXpq3HK+44gpnr4s/VIiIiEi1SxmVUtspiIiDL4PMq9Jnn31GSkoKW7ZsITY2lhEjRtCqlW9P0isuLmbr1q2EhISUmT916lSio6NJTEzk6NGjDBkyxBlr3LhxmWUbNmzo/Dk4OJjCwkIsyyI8PNzjOAnXNkqEhISQk5PjNefS615I/q5K8g8KCqJ+/foYY5zTRUVFFe5Lyfol++6rOXPmeMwxLy+PRo0a+dyWJxojIiIi1S7iqggiroqo7TREpIZZlsXjjz9OQkIC11xzDY8++ihPP/20z+v36dOH119/3Tn92WefAfan9a1btwZgxYoVlc6rffv2nDx5kt27dwP27VQHDhyocL1f/vKXHDlyxDndpEkTTp8+XWX5V9ReVe2LL9vxdoy/+uorOnToUKk83VGPiIiIiEgAyF2WQl5eXrlP56vTsmXLuPrqq523Y40fP57ExETS0tKIiooqs+zZs2fp2LGjc3ry5MnMnTuXP/zhD0RGRlJUVERkZKRzbMKkSZOYP38+AwYMqHReDRo0YNmyZUybNo2cnByKioqYNGlShRfXkZGR/PGPf8SyLIwxjBgxgri4OP7yl7+wfPnycstXNv877riDBQsWEBUVxWOPPVZt+9KzZ08WL17s3M7w4cPLLePtGKelpVXJI4ZNyZMD6oLs7Oy6k+xFIiMjg/bt29d2GlLLdB5IbZ8DCR8lADDj9hm1loPU/nkgNS87O5vQ0FDndE0XIpeiadOmERsbS0xMTG2ncsH8OQ/Onz/P4MGD2bRpE/Xqle3TcD3fSgsNDTWu83RrloiIVLu5u+Yyd9fc2k5DRMRvjz/+OGfPnq3tNGrN8ePHmTlzZrki5ELo1iwREal207pPq+0URESqxJVXXsmgQYNqO40q89Zbb/Haa6+VmdejRw/mz5/vdvl27drRrl27Ktm2ChEREal2uiVLROTiNGbMGMaMGVMr29atWSIiIiIiUuNUiIiISLVL/z6d9O/TazsNERG5iOjWLBER8VtYWBgAWVlZbuMxK2Ps+BT3cRGpfme2DbT/reJ2G/fZVMUtSqBQISIiIn7r0qWL9/iV3uMiIhJ4VIiIiIjftm/f7j0+2ntcRGpOVfVglPSwVKR58+Z07NiRoqIirr32Wv7yl784e1FLa9OmDd9++22V5FadNmzYwP79+5k2bRobNmzghhtuIDw8vEra/uyzzzhx4kSVfFmgN/v27ePUqVMXtJ39+/fz8ssv8+qrr/qdh8aIiIiIiEi1adSoEWlpaXz00Uc0a9aMN954o0a2W1hYWC1tvPTSS4wfPx6Ad955hy+++KLKtr937162bNlS6fUqa9++fRe8nU6dOvHdd9/xzTff+J2HChERERERqRG33XYbmZmZPi9/8uRJ7r//fnr37k3v3r3ZuXMnAJ988gl33nkn0dHR9O/fn4yMDAASExO59957ueuuuxg6dCipqakMHjyYBx54gFtvvZUJEyZgWRYA6enpDBo0iF69ejF8+HBOnDgBwODBg5k+fToxMTHlPvX/6quvaNCgAS1atGDXrl0kJyfz9NNPExUVxZEjR8qtW5n88/PzSUhIYM2aNURFRbFmzRoSEhKYOHEisbGx3HTTTaxbt45nnnmGyMhIRowYQUFBQYX7MnPmTPr06cMtt9zCjh07yM/PZ968eWW2446nYwwwcOBAj+tVRo0UIsaYa4wx7xtjPjfG7DfGxLlZxhhjXjLGfGWM+cwY07UmchMREf+Fh4d7vTUhfEk44Uuq5tYFEambioqK2L59O7GxsT6vM336dCZPnsz777/P8uXL+d3vfgdA+/btSU5OJjU1lSeffJJZs2Y51/nss89Yvnw5GzduBOxehoSEBHbt2sXXX3/Nzp07KSgo4IknnmD58uVs376dMWPGMHv2bGcbBQUFpKSk8Oijj5bJZ+fOnc4xcd27dyc2NpbZs2eTlpbG9ddfX27dyuTfoEEDZsyYwfDhw0lLS2P48OEAfP3116xbt46VK1fy8MMPEx0dzY4dOwgJCWHz5s0V7kthYSHbtm0jISGBuXPn0qBBA6ZOnVpuO668HeObb76ZHTt2+Pw+elJTY0QKgccty/o/Y0xT4BNjzBbLsj4vtUws0N7x6g686vhXREQuciWfvnmMn/EeF5FL17lz54iKiiIzM5Mbb7yR3r17+7xuSkoKBw8edE6fPn2a3NxccnJymDRpEocPH8YY4+wZAIiJiaFZs2bO6a5du9KmTRsAOnfuzLFjxwgNDeXAgQP86le/AqC4uJirrrrKuc7dd9/tNp/vv/+eli1bes259LoXkr+rfv36Ub9+fTp16kRRURH9+vUDoGPHjhw7doyMjAyv+3LXXXcBEBERwbFjx09o73oAACAASURBVLzmXpq3HK+44ooK/+77okYKEcuyMoFMx8+njTEHgDZA6UJkGLDcsvvLdhpjwowxrR3riojIRezAgQPe4+O9x0Wk5vg6yLyqlIwROXv2LCNGjGDJkiVMnDjRp3WLi4vZunUrISEhZeZPnTqV6OhoEhMTOXr0KEOGDHHGGjduXGbZhg0bOn8ODg6msLAQy7IIDw/3OE7CtY0SISEh5OTkeM259LoXkr+rkvyDgoKoX78+xhjndFFRUYX7UrJ+yb77as6cOR5zzMvLo1GjRj635UmNPzXLGHMdcDOwyyXUBig96uW4Y57bQqT0fWrinY6VgM4DqZlzoKJtZKDzsLbpb0FgCQkJKXMhXh3y8vK8xi3LIi8vj6CgIGbNmsW4ceMYM2YM9erVc7tcab169WLx4sU88sgjgD3I+qabbiIrK4uWLVuSl5fH8uXLnesWFBRQWFjobCc/P5+ioiLndGFhIQUFBVxzzTX8+OOPpKWl0a1bNwoKCjh06BDh4eEUFxdz/vx5t/t1/fXXk5SU5Iw1atSIU6dOOadd161s/g0bNiQrK6tcviXTpY9RZfclLy/PuX6TJk3KbMcdTzmC/eHTjTfeWG79nJwcfvjhB+d0+/btPbYPNVyIGGOaAEnAFMuyvJeTFahox8SWkZGhYyU6D0TngAA6DwJRdnb2vz+N77OJvLy8cp/OVzdjjHObt956KzfddBMbNmzg3nvvLbPcuXPn6Nr130OEJ0+ezPz58/nDH/5Anz59KCoqIjIykhdffJHf//73TJo0iYULFzJgwADnNurXr0+9evWc22vQoAHBwcHO6Xr16lG/fn0uv/xyli9fzrRp08jJyaGoqIhJkyYRERFBUFAQDRs2dHucYmJimDVrFg0bNsQYw3/+538SFxfHX//6V5YvX15u3crm37dvXxYvXky/fv147LHHnPmWtFf6WFZ2X0JCQpzr9+zZs8x23I0T8ZQjwK5du4iNjS13jC6//HKuueYa38+NkicHVDdjTH1gA7DZsqw/u4n/BUixLGulY/oLIKb0rVnZ2dk1k+wlRP/pCOg8kOo/B+Li7GeQLFy40H18qyPez31caob+FgSe7OxsQkNDndO1UYhcaqZNm0ZsbCwxMTG1ncoF8+c8OH/+PIMHD2bTpk3lerVcz7fSQkNDjeu8mnpqlgH+BzjgrghxWAc84Hh6Vg8gW+NDRETqhmXLlrFs2TLP8X3LWLbPc1xEpK54/PHHOXv2bG2nUWuOHz/OzJkzyxUhF6Kmbs3qCdwP7DXGpDvmPQlcC2BZ1mvARmAQ8BVwFniwhnITERE/LViwwHu8r/e4iEhdceWVVzJo0KDaTqPKvPXWW7z22mtl5vXo0YP58+e7Xb5du3a0a9euSrZdU0/NSgPKdce4LGMBj9REPiIiUrXGjRvnPd7Ze1xERGrHmDFjGDNmTK1sW9+sLiIiIiIiNU6FiIiI+C05OZnk5GTP8cPJJB/2HBcRkcBT498jIiIil55Ro0YB9nPn3cbXOeJT3MdFpPqFLQirlnb1ey0XSj0iIiLitwEDBjBgwADP8esHMOB6z3ERubRt2LCBsLAwvvzyywqXTUxMJDPzwh+cmpqaysiRIz3Gp0+fTocOHSguLgbswdpRUVFERUVxxRVXEBkZSVRUFPHx8SQmJtKuXTtnPCoqioMHD3rdfkJCAosWLQLsbydPSUnxuOyGDRsqbO9Sph4RERHx26pVq9zOr65PYEXkwlVVD0Zlfr+TkpK4/fbbWb16NU8++aTXZVesWEHHjh1p3bq1vymWU1xczIYNG2jTpg1paWnccccdZQZrd+7cmfXr19OiRQvALoqGDx/OvHnzLmh7Tz31lNf4O++8w8CBAwkPDy8XKywsrJJH5F7M1CMiIiIiItUmNzeXnTt3smjRItasWVMmtmDBAiIjI+nZsyfx8fGsXbuW9PR0JkyYQFRUFOfOnaNz58789NNPAHz66acMHjwYgE8++YQ777yT6Oho+vfvT0ZGRoW5pKam0qFDBx566CGSkpKqbB/nz5/PLbfcwsCBA/nqq6+c8ydNmsTatWsBiI+Pp3v37kRGRvLHP/6RXbt2kZyczNNPP01UVBRHjhxh8ODBTJ8+nZiYGF599dUqy+9idWmXWSIiIiJSqzZu3Ejfvn254YYbaN68Oenp6URERLBlyxY2btzI1q1bueyyy/j5559p1qwZr7/+On/605+4+eabvbbbvn17kpOTqVevHikpKcyaNYs333zT6zpJSUmMGDGCQYMGMXv2bAoKCqhfv77XddasWcNHH33knN6yZQuNGjVyTqenp7NmzRpSU1MpLCykV69eRERElGnj1KlTbNiwgX/9618YY8jKyiIsLIzY2FgGDhzIsGHDnMsWFBR4vZ3rUqJCRERE/BYWZt+i4WmwuogErqSkJCZOnAjA8OHDWb16NREREaSkpHDfffdx2WWXAdCsWbNKtZuTk8OkSZM4fPgwxhgKCgq8Lp+fn8+WLVuYM2cOTZs25ZZbbuG9995j4MCBXter6NasHTt2MHjwYOd+xMbGllvm8ssvp2HDhvzXf/0XAwYM8LrNu+++22s+lxIVIiIiIiJSLX7++Wc++OAD9u/fjzGG4uJijDHMnj3b5zbq1avnHFiel5fnnD9nzhyio6NJTEzk6NGjDBkyxGs77733HtnZ2URGRgJw7tw5GjVqVGEhUhXq1avHtm3b2L59O2vXrmXJkiWsX7/e7bKNGzeu9nwuFipERETEb+oJEak7avIhEmvXrmXkyJEsWLDAOW/QoEHs2LGD3r178/zzz3PPPfeUuTWrSZMmnD592rn8tddeS3p6OnfeeSfr1q1zzs/JyXEOaF+xYkWFuSQlJfHSSy/x61//GoAzZ87QpUsXzp496+zNuBCRkZFMnjyZxx57jMLCQjZt2sSDDz5YZpnc3FzOnTtH//796d69u/PWLdd9DTQarC4iIiIi1WL16tXleiqGDh1KUlIS/fr1IzY2lt69exMVFeV85O3o0aN57LHHnIPVp02b5hzAHRwc7GwnLi6OWbNmER0dTVFRkdc8zp49y9atW+nfv79zXuPGjenRowebNm3yuu6aNWvKPL53165dZeIREREMHz6cqKgo7rnnHrp27VqujdzcXEaOHElkZCSxsbHMmTMHgBEjRrBo0SKio6M5cuSI1zwuRcayrNrOwWfZ2dl1J9mLREZGBu3bt6/tNKSW6TyQ2joHXD951Ref1S79LQg82dnZhIaGOqfz8vIICQmpxYzkYlBd54Hr+VZaaGiocZ2nHhEREfHbyJEjvX6BmIiIiCuNEREREb9t3ry5tlMQEZE6RoWIiIj4beXKlbWdgoiI1DEqRERExG/unpvvTsmYEY0VERERjREREREREZEap0JERET8tnTpUpYuXeoxvqDvAhb0XeAxLiIigUe3ZomIiN+mTJkCwLhx49zH35tSg9mIiEhdoB4RERHx29ixYxk7dqzn+E1jGXuT57iIXLrmz59Pjx49iIyMJCoqio8//rjcMpMmTWLt2rW1kF3lnDt3jkGDBlFUVMTRo0d5++23q7T9F154oUrb82ThwoUXvO6wYcPIyqqacX4qRERExG8LFy70+h/bwn4LWdjvwv/jE5G6affu3WzevJnt27ezY8cO1q5dS5s2bWpk2xV923pFCgsLy8176623uOuuuwgODubYsWOsXr3a53V98ec///mC1qssfwqRkSNH8sYbb1RJHipERERERAJEq1atCAsLKzNv5MiRhIWFkZyc7Jy3dOlSwsLCiIuLc87LzMwkLCyM8PBwn7d34sQJmjdvTsOGDQFo0aIFrVu39nn9l156id69exMZGclzzz3nnD969Gh69epFjx49yoxPa9OmDU899RQ9e/Zk9+7ddO7cmeeee4477riDyMhIvvzySwDOnDnDI488Qp8+fYiOjuadd94BIDExkXvvvZe77rqLoUOHlsvn7bffZtCgQQA8++yzfPTRR0RFRbF48WK361Ym//j4eM6dO0dUVBQTJkzg6NGj3HrrrUyaNIlbbrmFCRMmkJKSwoABA+jatSuffPJJhfsyZswYRowYQdeuXXnmmWec28nLy3NuxxNPx3jQoEEkJSX59P5VRGNERETEb5mZmQAeLzAyczNrMh0RuUj06dOH559/nltuuYWYmBjuvvtuoqKifFp327ZtHDp0iG3btmFZFqNGjeLDDz+kZ8+eLF68mGbNmnHu3Dn69OnD0KFDad68OWfOnKFbt27MmTPH2U6LFi344IMPeOONN1i0aBGLFi3ihRde4I477mDx4sVkZWXRt29fYmJiAPjss8/48MMPadasWZl88vPz+frrr2nbti0AM2fO5OWXX2bVqlWAfeFfet3K5h8fH8+SJUtIS0sD4OjRoxw+fJilS5fSoUMHevfuzdtvv82mTZvYuHEjL7zwAitWrPC6L3v37uWDDz6gYcOGdOvWjd/+9rfEx8fz+uuvO7fjiadjHBYWxvnz5zl16hTNmzf36b30RIWIiIj4rUOHDgAe7xvu8EaHmkxHRDw4ceIEISEhZeaVXEiXNm7cuHIPn2jdunWlxwY0adLEeVtWamoqDz30EDNnzuS+++6rcN1t27axbds2oqOjAfuT/0OHDtGzZ09ee+01NmzYAMC3337LoUOHaN68OcHBweV6Mu666y4AIiIiWL9+vbPt5ORkFi1aBMD58+c5fvw4ADExMeWKEICffvqJ0NBQrzmXXvdC8nfVtm1bOnXqBEB4eDi9evXCGEOnTp04duxYhfvSq1cvZ87h4eF88803XH311V73oYS3HK+44goyMzNViIiISO1r1aqV93hjO37izImaSEdELiLBwcFER0cTHR1Np06dWLFihU+FiGVZPPbYYzz44INl5qemprJ9+3a2bNnCZZddxuDBg8nLywMgJCSE4ODgMsuX3BYWHBzsHLthWRbLly+nffv2ZZb9+OOPady4sdt8GjVq5NyOJ6XXvZD8XZXkDhAUFOScNsY4x8B425fS65fe/4pUlGNeXh6NGjXyqS1vNEZERET8dvDgQQ4ePOg5PuEgByd4jovIpSkjI4NDhw45p/fu3cu1117r07p9+/blrbfeIjc3F4DvvvuOH3/8kZycHEJDQ7nsssv48ssv3T6Fy5e2X3/9dSzLAmDPnj0VrhMWFkZxcbHzgrxp06acPn26SvOvV68eBQUF1b4v9evX97odbzlalsUPP/zg8/vojQoREREREakWZ86cYdKkSXTv3p3IyEgOHjzI9OnT3S47ZcoUOnbsSMeOHbnzzjvp06cPv/71r+nfvz+RkZGMHTuW3Nxc+vXrR1FREbfddhvx8fF069at0nlNnTqVgoICevbsSY8ePcoMJPemd+/e7Ny5E4BOnToRHBzsHPPh6kLyHzduHD179vQ6iLwq9mXMmDFet+Mtx/T0dLp160a9ev7fWGVKqqe6IDs7u+4ke5HIyMgo11UngUfngdTWORC2wH46T9aULLfTUrP0tyDwZGdnlxnXkJeXV26MiPguPT2dV155hddff722U/GLP+fBtGnTGDRoEL169SoXcz3fSgsNDTWu8zRGRERE/FbyH9L27dvdx1eU/w9LRKSuiYiIIDo6mqKionJjUQJFx44d3RYhF0KFiIiI+K2ie5L3/FDxPcsiInXB/fffX9spVJlTp065/b6UdevWeXwi1tixY6ts+ypERETEbykpKd7jo+x4zMqYas9FRER807x58wq/T6Q6qRARERG/RUREeI9f5T0uIiKBR0/NEhERERGRGqceERER8VtCQgIAM2bMcB//KKEm0xERkTpAPSIiIuK3uXPnMnfuXM/xXXOZu8tzXEQubRs2bCAsLIwvv/yywmUTExPJzMy84G2lpqYycuRIj/Hp06fToUMHiouLAXjrrbeIiooiKiqKK664gsjISKKiooiPjycxMZF27do541FRUV6/vFUqRz0iIiLit2nTpjl/LvmukDLx7na8pBjR94mIBJakpCRuv/12Vq9ezZNPPul12RUrVtCxY0dat25d5XkUFxezYcMG2rRpQ1paGnfccQdjxoxhzJgxAHTu3Jn169fTokULwC6Khg8fzrx586o8F1GPiIiIVIEZM2Z4vC0LYMbtM5hxu+e4iFy6cnNz2blzJ4sWLWLNmjVlYgsWLCAyMpKePXsSHx/P2rVrSU9PZ8KECURFRXHu3Dk6d+7MTz/9BMCnn37K4MGDAfjkk0+48847iY6Opn///mRkZFSYS2pqKh06dOChhx4iKSmp6ndWKkWFiIiIVIusKVnOl+s8EakdrV5rVa7XcuTakYQtCCP5cLJz3tK9SwlbEEbc1jjnvMzcTMIWhBG+JLxS29y4cSN9+/blhhtuoHnz5qSnpwOwZcsWNm7cyNatW/nwww+Ji4tj2LBhREREsGTJEtLS0mjUqJHHdtu3b09ycjKpqak8+eSTzJo1q8JckpKSGDFiBEOGDOHdd9+loKCgwnXWrFlT5tasc+fO+b7z4pVuzRIREb+VXFh4eoxv+veOuB7jKxJwkpKSmDhxIgDDhw9n9erVREREkJKSwn333cdll10GQLNmzSrVbk5ODpMmTeLw4cMYYyosKvLz89myZQtz5syhadOm3HLLLbz33nsMHDjQ63q6Nav6qBARERG/xcTEAJCV5b63o+SLDNUbIlK7Tkw8QUhISJl5q4atKrfcuM7jGNd5XJl5rZu0rvTv8M8//8wHH3zA/v37McZQXFyMMYbZs2f73Ea9evWcA8vz8vKc8+fMmUN0dDSJiYkcPXqUIUOGeG3nvffeIzs7m8jISADOnTtHo0aNKixEpPro1iwREfFbly5d6NKli+f4lV3ocqXnuIhcmtauXcvIkSPZt28fe/fuZf/+/Vx77bXs2LGD3r17k5iYyNmzZwG7aAFo0qQJp0+fdrZx7bXXOntd161b55yfk5PjHNC+YsWKCnNJSkripZdeYu/evezdu5c9e/bw/vvvO7cvNU+FiIiI+G379u1s377dc3z0draP9hwXkUvT6tWry/VUDB06lKSkJPr160dsbCy9e/cmKiqKRYsWATB69Ggee+wx53iMadOmMX36dGJiYggODna2ExcXx6xZs4iOjqaoqMhrHmfPnmXr1q3079/fOa9x48b06NGDTZs2eV3XdYzIrl27KnsYxANjWVZt5+Cz7OzsupPsRSIjI4P27dvXdhpSy3QeSE2eA748mleP760d+lsQeLKzswkNDXVO5+Xllbs1SwJPdZ0HrudbaaGhocZ1nnpERERERESkxqkQERERv4WHhxMe7vmRnuFLwiv9yE8REbm06alZIiLitxMnTniPn/EeFxGRwKNCRERE/HbgwAHv8fHe4yJSPYKCgsjPz6dBgwa1nYpc4vLz8wkKqtzNVipERETEbyWP0PQYb+I9LiLVo0mTJuTm5jq/DTwnJ4fLL7+8lrOS2lYd50FQUBBNmjSp1DoqREREREQuUcYYmjZt6pz+4YcfuOaaa2oxI7kYXCzngQari4iI3+Li4oiLi/Mc3xpH3FbPcRERCTwqRERExG/Lli1j2bJlnuP7lrFsn+e4iIgEHt2aJSIifluwYIH3eF/vcRERCTwqRERExG/jxo3zHu/sPS4iIoFHhYiIiFSJsAVhtZ2CiIjUIRojIiIifktOToYvvMQPJ5N8OLnmEhIRkYueekRERMRvo0aNsn+Ih6wpWeXj6+y4u5iIiAQmFSIiIuK3AQMGsPnIZs/x6wfUYDYiIlIXqBARERG/rVq1yusYkVXDVtVgNiIiUhdojIiIiIiIiNQ4FSIiIiIiIlLjVIiIiIjfwsLCIN5LfEGYHu8rIiJlqBARERG/qMAQEZELocHqIiLiv3jvYT22V0REXKkQERGRKnEhxUZJb4oKFRGRwKNbs0REREREpMapEBEREf+tgJEjR3oMj1w7kpFr/x3PmpKlXhARkQCnW7NERMR/X8LmLz1/s7q3b10XEZHApEJERET8NwpWDl3pMewtJiIigUmFiIiI+O+XEBsb6zEc+wvPMRERCUwaIyIiIiIiIjVOhYiIiPjvY1i6dKnH8NK9S1m613NcREQCjwoRERHx3waYMmWKx/CU96Yw5T3PcRERCTwaIyIiIv7rCmM7j/UYHnuT55iIiAQmFSIiIuK/obBwykKP4YX9PMdERCQw6dYsERERERGpcSpERETEfzmQmZnpMZyZm0lmrue4iIgEHhUiIiLivz9Dhw4dPIY7vNGBDm94jouISODRGBEREfFfE2jVpJXHcKvGnmMiIhKYVIiIiIj//gAHpxz0GD44wXNMREQCk27NEhERERGRGqdCREREREREapwKERER8d9foFevXh7DvVb0otcKz3EREQk8GiMiIiL+y4Q9mXs8hvf84DkmIiKBSYWIiIj477eQMjrFYzhllOeYiIgEJhUiIiLiv/8HERERHsMRV3mOiYhIYNIYERERERERqXEqRERExH/vQ0JCgsdwwkcJJHzkOS4iIoGnRgoRY8xfjTE/GGP2eYjHGGOyjTHpjtczNZGXiIhUke0wd+5cj+G5u+Yyd5fnuIiIBJ6aGiOyFHgZWO5lmVTLsobUTDoiIlKlesG0HtM8hqd19xwTEZHAVCOFiGVZHxhjrquJbYmISC3oDTOmzPAYnnG755iIiASmi2mMyO3GmD3GmGRjTKfaTkZERERERKrPxfL43v8D2lqWlWuMGQT8E2jvbYWMjIwaSexSoGMloPNAqvkc+A7WrVtHhw4d3IYPZB8AoEOo+7jOz5qjYy06BwRq5jxo397r5TzGsqxqTwLAcWvWBsuybvJh2a+BbpZlnSw9Pzs7u2aSvYRkZGRUeBLIpU/ngVTnORC2IAzi7Z+zsrI8LwNkTcnyab5UD/0tEJ0DArVzHoSGhhrXeRdFj4gxphXwvWVZljHmNuxbxn6q5bRERMRXraHLlV08hr3FREQkMNVIIWKMWQnEAC2NMceBmUB9AMuyXgN+DUwyxhQC54B7rZrqqhEREf89DNunbPcY3j7ac0xERAJTTT01a1QF8ZexH+8rIiIiIiIB4GJ6apaIiIiIiAQIFSIiIuK/+RAeHu4xHL4knPAlnuNhC8KcA9dFRCQwXBSD1UVEpI7LhRO5JzyGT5zxHBMRkcCkQkRERPz3GByYcMBj+MB497GSx/aqN0REJPCoEBEREf9dDq1bt/YYbt3Ec0xERAKTxoiIiIiIiEiNU4+IiIhUitvbqNZB3JE4Fi5c6HaduK1xACzs5z4uIiKBRz0iIiLiv/+DZcuWeQwv27eMZfs8x0VEJPCoR0RERPw3BBb0W+AxvKCv55iIiAQmFSIiIuK/bjBu3DiP4XGdPcdERCQw6dYsERG5aOiLDUVEAocKERER8d8XkJyc7DGcfDiZ5MOe4yIiEnh0a5aIiPhvJYxaOYqsrCy34VHrRgH//gJDV/piQxGRwKNCRERE/HcjDLh+gMewt5iIiAQmFSIiIuK/0bBqyiqP4VXDPMdERCQwaYyIiIiIiIjUOBUiIiIiIiJS41SIiIiI/+IhLMzzQHM9lldERFypEBERERERkRqnweoiIuK/eM+P5gXvMRERCUwqRERE5IKouBAREX/o1iwREREREalxKkRERMRvI0eOZOTIkZ7ja0cycq3nuIiIBB7dmiUiIn7bvHmz9/gR73EREQk8KkRERMRvK1eu9B4f6j0uIiKBR4WIiIj4LTY21nv8F97jIiISeDRGREREREREapwKERER8dvSpUtZunSp5/jepSzd6zkuIiKB54JuzTLGNAKKLcs6X8X5iIhIHTRlyhQAxo0b5z7+niPe2X1cREQCj0+FiDFmPvC/lmXtNsYMBlYDljFmpGVZ66s1QxERueiNHTvWe/wm73EREQk8vvaI3Ac84/j5GWAMkA28CKgQEREJcAsXLvQe7+c9LiIigcfXQuQyy7LOGmNaAL+wLCsJwBjTtvpSExERERGRS5WvhciXxpj7gBuALQDGmJbAuepKTERE6o7MzEwAWrdu7T6e64g3cR8XEZHA42shMhlYCBQADznmDQDerY6kRESkbunQoQMAWVlZ7uNvOOJT3MddhS0Iq9TyIiJS9/hUiFiW9S8g0mVeIpBYHUmJiEjd0qpVK+/xxt7jIiISeHx+fK8x5k7gXuBKy7LuMsZ0Ay63LGtbtWUnIiJ1wsGDB73HJ3iPlyjpASnpERERkUuXT19oaIx5FHgVyADucMw+B/ypmvISEREREZFLmK/frD4F6GdZ1n8DxY55B4FfVktWIiIiIiJySfO1EGkKfOP42XL8Wx/Ir/KMRESkzunVqxe9evXyHF/Ri14rPMdFRCTw+DpG5ANgOjCn1LzfAe9XeUYiIlLn7Nmzx3v8B+9xEREJPL4WIo8C640xE4CmxpgvgNPAkGrLTERE6oyUlBTv8VHe4yIiEnh8fXxvpjHmVuA24Frs27R2W5ZV7H1NEREJBBEREd7jV3mPi4hI4PH58b2WZVnALsdLRERERETkgnksRIwx3/DvgekeWZZ1bZVmJCIidU5CQgIAM2bMcB//yBG/3X1cREQCj7cekTE1loWIiNRpc+fOBTwXInN3OeIqRERExMFjIWJZ1vaaTEREROquadOmeY939x4XEZHA49MYEWNMA+CPwCjg/wHfAX8H5liWlVd96YmISF3gqSfEGVdPiIiIuPB1sPqr2N+i/jvgKNAWeBJoAzxUPamJiIiIiMilytdC5FdAO8uyshzTnxtjdgFfoUJERCTgpaenA54f45v+vSOux/iKiIiDr4XICeAyIKvUvEZAZpVnJCIidU5MTAwAWVlZ7uMrHfEp7uMiIhJ4fC1E3gQ2GWMWAceBa4BHgOXGmD4lC1mWta3qUxQRkYtB2IIwj7EuXbp4XbfLld7jIiISeHwtRB52/Puky/yJ87ia6QAAIABJREFUjhfY3znyi6pISkRE6pbt270/aHH7aD2IUUREyvKpELEs6/rqTkREROoG3V4lIiJVIai2ExARERERkcDjUyFijOlijNlmjDlljMl3vAqMMfnVnaCIiFz8wsPDCQ8P9xxfEk74Es9xEREJPL6OEVkJJGF/j8i56ktHRETqohMnTniPn/EeFxGRwONrIdIKeMayLKs6kxERkbrpwIED3uPjvcdFRCTw+FqILANGA4nVmIuIiNRRrVu39h5v4j0uIiKBx9dC5L+Bj4wxTwLflw5YltXH/SoiIiL+Kf3dJXpal4jIpcXXQmQ1cAT4BxojIiIiLuLi4gBYuHCh+/hWR7yf+7iIiAQeXwuRCKCFZVl6SpaIiJSzbNkywHMhsmyfI+5jIVK698PbN7qLiEjd5Wshkgp0BNKrMRcREamjFixY4D3e13tcREQCj6+FyBHgXWPMPyg/RuSZKs9KRETqlHHjxnmPd/YeFxGRwONrIXIZ8A7QALim+tIREREREZFA4FMhYlnWg9WdiIiI1F3JyckAxMbGuo8fdsR/4T4uIiKBx9ceEQCMMU2BloApmWdZ1uGqTkpEROqWUaNGAZCV5f4Ru6PWOeJ6BK+IiDj4VIgYYzpif5lhF8DCLkRKvmU9uHpSExGRumLAgAHe49d7j4uISODxtUfkFeB9oDf2wPXrgARgR/WkJSIidcmqVau8x4d5j4uISODxtRDpAtxpWVaBMcZYlpVtjJkK7APeqr70RERERETkUhTk43J5QH3HzyeNMdc61m1RLVmJiIiIiMglzddCJBX+f3v3HiVXVeZ9/PcICpIg5Q0S7qAZOkpMeL0w6DjJAE5oBcLrMIbgi2kU1DVeul1eYpilMo4zscXRjktnfCVKtwghI+NIRJoMl9VR18S8XOxwMdHGKJKYBLkUkghixuf945wK1Z1zdlf3qTrVVef7WatXV53fqVO7Thekntpn7623xbevlzQoab2k2xvRKABAaymVSiqV0ldAL/WVWCEdADBKrdP3vq3q7mWS7pc0XdI3G9EoAAAAAO1tQtP3SpK7/8nMbnT3xxvRIABA60mbtndfzrS9AIAxgpdmmdk7zGxh1f3XmNk2ReNEtpjZSQ1vIQAAAIC2M94YkY9I2ll1/2uSbpH0Kkm3SbqiQe0CAAAA0MbGuzTrGEn3SpKZHSNpjqQz3f0xM/u4pAca3D4AQAtYvHixpPT1RBbfEOesJwIAiI1XiOyV9DxF0/e+XtIWd38szn4v6fkNbBsAoEWsW7cunP8ynAMAime8QmS9pH8yswFJH5D0vaqsQ6Mv2wIAFNTq1avD+bnhHABQPOMVIt2Srpb0bkkbJPVWZRdJurlB7QIAtJDOzs5wfmI4BwAUT7AQcfftkk5PyT7ekBYBAAAAaHu1rqwOAECq/v5+9ff3p+f39qv/3vQcAFA8E17QEACAsXp6eiRJXV1dyfltcT4nOQcAFA+FCAAgs6VLl4bzk8M5AKB4UgsRM7vC3T8a3z7d3W/Pr1kAgFaycuXKcH5mOAcAFE9ojMi7q25/t9ENAQAAAFAcoUuzNpnZ9ZJ+KukgM/t00k7u/smGtAwA0DJ27NghSZo5c2ZyvjvOpyfnAIDiCRUi5yvqFTlOkkk6JmEfb0SjAACtZfbs2ZKkcrmcnK+K857kHABQPKmFiLs/LOkzkmRmB7r7xbm1CgDQUmbMmBHOp4VzAEDx1DRrlrtfbGYvlHSOpKMkbZd0o7s/Vsvjzewbks6W9LC7n5yQm6SVkt4s6feSutz97tpeAgCgkUp9pXH32bJlSzi/NJwDAIqnpgUNzew0Sb+Q9F5Jr5L0HkkPxNtr0S/prEDeKWlW/PNuSf9W43EBAAAAtKBa1xHpk/R37n5dZYOZLZb0JUmvHe/B7v4DMzs+sMsiSd90d5f0YzMrmdlMd99RY/sAAA3W7PEdlZ6ZZrcDAFAftRYifybp38dsu17SV+vUjqMkPVR1f1u8LbUQGRkZqdNTtz/OFSTeB8j+Hgg9/qKLLpIkXX311cn5D+P8jcl5vdqB8XH+wHsAUj7vg1mzZgXzWguREUkXSLq2atvfKrpcqynGe2GIjIyMcK7A+wB1eQ+EHl8ZI5K2z5bvh/OQSg9IpUeE9/Lk8f8C8B6ANHXeB7UWIj2SbjSzD0p6UNLxisZznF2ndmzX6OmBj463AQBawNDQUDhfEs4BAMVT66xZ/21mL5P0FklHSvqepJtqnTWrBmslvd/MrpN0qqQnGB8CAK1j3rx54fyIcA4AKJ5ae0Tk7o9L+tZknsTMVktaIOklZrZN0qckPTc+7lcl3aRo6t4HFE3fy5olAAAAQBuruRDJwt2XjJO7pPfl0RYAQP2tWLFCkrR8+fLkfEOcn5acAwCKp6Z1RAAACOnt7VVvb296vrFXvRvTcwBA8eTSIwIAaG/Lli0L56eGcwBA8dRciJjZce7+YCMbAwBoTWmXZO3LuSQLADDGRC7N+okkxVP4AgAAAMCkBXtEzOwuSXcpKkIOiDdfLulLjW0WAKCVDA8PS0qfxnd4V5wzjS8AIDbepVnnS3q1pNdIOsTM7pZ0kJn9laS73f2JRjcQADD1LViwQJJULpeT89Vx3pOcAwCKZ7xC5AB3v17S9Wb2bkmLJP1M0gcknWJme929+evDAwCaau7cueH88HAOACie8QqRa8zsWEk/lXSwpBdKetrd3ypJZvaiBrcPANAkpb5SzfuuX78+nF8YzgEAxRMsRNz9VDM7UNIcST+S9GVJh5rZv0m6O/55rOGtBAAAANBWxp2+1933SvqJmT3j7n9pZmVJQ4rGjiyWdGZjmwgAaCbGdQAAGmEiCxp+KP7t7r5G0poGtAcA0II6OjokSVu2bEnOr4zzS5NzAEDx1FyIuHt/fPPExjQFANCqdu7cGc73hHMAQPFMpEdEkuTujzeiIQCA1rV58+Zwfkk4BwAUz4QLEQAAxpo5c2Y4nx7OAQDF85xmNwAAAABA8VCIAAAy6+7uVnd3d3p+a7e6b03PAQDFQyECAMhsYGBAAwMD6fl9Axq4Lz0HABQPY0QAAJn19fWF8zPCOQCgeChEAACZdXV1hfM54RwAUDxcmgUAAAAgdxQiAIDMBgcHNTg4mJ5vHdTg1vQcAFA8XJoFAMhsyZIlkqRyuZycr43znuQcAFA8FCIAgMwWLlwYzk8I5wCA4qEQAQBktmbNmnC+KJwDAIqHQgQAUBfTly6Y1ON2DwzVtR0AgNbAYHUAAAAAuaNHBACQWalUkiTtXTQvsYej1Bfl1YPVJ9uDAgBoD/SIAAAAAMgdPSIAgMzK5XKwh4NpewEAY9EjAgAAACB3FCIAAAAAckchAgDIbPHixVr0463p+Q2LtfiGxTm2CAAw1TFGBACQ2bp168L5L8P5RCTNwAUAaD0UIgCAzFavXq2D+y5Lz89dnWNrAACtgEIEAJBZZ2enpl/Xm56f2Jn5OSo9IJUeEQBAa2OMCAAAAIDcUYgAADLr7+/Xlb96JD2/t1/99/bn1yAAwJTHpVkAgMx6enokSZce/5Lk/LYo75rTtV8WWggx0SkT2x0AMDVRiAAAMlu6dKmeO3Rjen7y0ro/J7NnAUBroxABAGS2cuVKTV+6KT0/c+V+23YPDE3oOSbccwIAmNIYIwIAaCnlnjK9IADQBugRAQDskzQ1bi09Eb956o/6naQjn//cxHzH7h2SpJnTZ2ZpHgCgjVCIAAAyO/a/7pck7V00LzGfvWq2JMZzAACeRSECANhPuae8ryeklrEcMzo6gvvOmDajTi0DALQLChEAQGZbtmwJ55eGcwBA8TBYHQAAAEDuKEQAAAAA5I5CBACQ2fz58zV//vz0/Nr5mn9teg4AKB7GiAAAMtu0KX0xQ0na9HA4BwAUD4UIACCzoaGhcL4knAMAiodCBACQ2bx5yeuH7MuPCOcAgOJhjAgAAACA3FGIAAAyW7FihVasWJGeb1ihFRvScwBA8VCIAAAy6+3tVW9vb3q+sVe9G9NzAEDxMEYEAJDZsmXLwvmp4RwAUDwUIgCAzJYvXx7OTwvnAIDi4dIsAAAAALmjEAEAZDY8PKzh4eH0fNewhnel5wCA4uHSLABAZgsWLJAklcvl5Hx1nPck5wCA4qEQAQBkNnfu3HB+eDgHABQPhQgAILP169eH8wvDOQCgeBgjAgAAACB3FCIAAAAAckchAgDIrKOjQx0dHen5lR3quDI9BwAUD2NEAACZ7dy5M5zvCecAgOKhEAEAZLZ58+Zwfkk4BwAUD4UIACCzmTNnhvPp4RwAUDwUIgCAljJ96YLoxilj7ifYPTDU6OYAACaJweoAgMy6u7vV3d2dnt/are5b03MAQPHQIwIAyGxgYECStHLlyuT8vjg/MzmvxX69G32l5O0K95IAAKYGChEAQGZ9fX3h/IxwDgAoHgoRAEBmXV1d4XxOOAcAFA9jRAAAAADkjkIEAJDZ4OCgBgcH0/Otgxrcmp4DAIqHS7MAACrFA78na8mSJZKkcrmcnK+N857kPItK2xtxbABA41CIAAAyW7hwYTg/IZwDAIqHQgQAsM9kexXWrFkTzheF88motDVrbw4AoDkYIwIAAAAgdxQiAAAAAHJHIQIAyKxUKqlUSr9EqtRX4hIqAMAoFCIAAAAAcsdgdQBAZmnT9u7LmVoXADAGhQgAIHelq7ZnPkb54qPq0BIAQLNQiAAAJqwehQQAoNgoRAAA2V35d9HvS/81Od8T59NG55Pp1aAIAoD2QCECAJi0SiFR+tD6UffHKvWFcwBA8VCIAAAyW716dTg/N5wDAIont0LEzM6StFLSAZJWuftnx+Rdkq6QVOlz/7K7r8qrfQDQ7qYvXZAenlLDPgGdnZ3h/MRwDgAonlwKETM7QNJXJL1J0jZJd5jZWnf/6Zhd17j7+/NoEwAAAIDmyatH5HWSHnD3rZJkZtdJWiRpbCECAGiw3QND+26PXe28OpuI/v5+SVJXV1dyfm+cz0nOAQDFk9fK6kdJeqjq/rZ421h/Y2b3mNn1ZnZMPk0DAGTV09Ojnp6e9Py2HvXclp4DAIpnKg1W/56k1e7+BzN7j6QBSaen7TwyMpJbw1od5woS7wM8K+m9cMdb7kjNkh0yav/zzjsv+PjzjgnnEzP6uSuq75+SsA0Rzgl4D0DK530wa9asYJ5XIbJdUnUPx9F6dlC6JMndH626u0rS50IHHO+FITIyMsK5Au8DjPoHJ+m9MOH3x4+2j3pc5dKsNP2zwnmW566oy+tqc/y/ALwHIE2d90Fel2bdIWmWmZ1gZs+TdIGktdU7mNnMqrvnStqcU9sAAG2g1Ffab8wLAGDqyqVHxN33mtn7Ja1TNH3vN9z9fjP7tKQ73X2tpA+a2bmS9kp6TFJXHm0DAGS3Y8cOSdLMmTOT891xPj05n4y0FdZLV23X3nH2kVhcEQCaLbcxIu5+k6Sbxmz7ZNXt5ZKW59UeAED9zJ49W5JULpeT81Vx3pOcZ3LY/dHvJ15Z/2MDABpmKg1WBwC0qBkzZoTzaeF8ItJ6Mkp9VflQ+r6hXhIAQH4oRACgoKo/kNfy4Xz7ce+quh393nN79Puufz0+vn9W4mPvelk4r5h2+s3jtgMA0B7yGqwOAAAAAPvQIwIAqGngdqX3oxG9FuP1lAAA2g89IgCAzObPn6/58+en59fO1/xr03MAQPHQIwIAyGzTpk3h/OFwDgAoHgoRAEBmQ0ND4XxJOAcAFA+FCAAgs3nz5oXzI8J5o0xfumC/bZXFDitT/FbbPZCwEQDQEIwRAQAAAJA7ekQAAJmtWLFCkrR8+fLkfEOcn5ac11uoZ6OyZkr1TGFJPScAgMaiRwQAkFlvb696e3vT84296t2YngMAioceEQBAZsuWLQvnp4ZzAEDxUIgAADJLuyRrX57TJVkAgNZBIQIAbSRthfIjJe1aenB0J95n+3HSUfeEHwcAQKMwRgQAkNnw8LCGh4fT813DGt6VngMAioceEQBoQ9NOv3nU/ZGREZ3ymUslPTujVDR71CsT95+oBQsWSJLK5XJyvjrOe5Lzinr0zCQdI+vrAwDUH4UIALSoyjS01bYfl5Ydsm8hv6THZTV37txwfng4BwAUD4UIALSovUNv329bZRxIUtZI69evD+cXhvO69FjcU9rvWIx9AYCpi0IEAAqmeiG/Ul8TGwIAKDQKEQBocaNWEY97AMauLD4yMqJZs2bl1ygAAMbBrFkAgMw6OjrU0dGRnl/ZoY4r03MAQPHQIwIAyGznzp3hfE84b4bqQfsTHchffXkbAGByKEQAAJlt3rw5nF8SzgEAxUMhAgDIbObMmeF8ejjPU2JvxlAgq9KIqY8BoKgYIwIAAAAgdxQiAIDMuru71d3dnZ7f2q3uW9NzAEDxUIgAADIbGBjQwMBAen7fgAbuS88BAMXDGBEAQGZ9feGVEfvOYOVEAMBoFCIAUEClvlJdj9fV1RXO54RzAEDxcGkWAAAAgNzRIwIABVbuKdflOIODg5Kkzs7O5HxrnJ+YnNdTpbenXq8NANAYFCIAgMyWLFkiSSqXkz/8L1kb5xQHAIAYhQgA5GzP7WfV5zhLD45u1Ol4WSxcuDCcnxDO66FS5CSNfxnvnNd6LrcfVznes9umnX5z7Y0EAOxDIQIAyGzNmjXhfFE4BwAUD4UIADRJ1m/Spy9dIEnaPTCUvTFtqtZzXDmXE7Gr0osCAJgUZs0CAAAAkDt6RAAAmZVK8UxVKYPVp/pMVrX2KpWu2i5J2jv09ga2BgCKgR4RAAAAALmjRwQAkFlaT8i+fIr2hNRDpZdkIsoXH9WAlgBAa6FHBAAAAEDu6BEBAKBG+3oyhhK21WAyvScA0K4oRACgibJ8MN1bx3ZktXjxYknp64ksviHOWU8EABCjEAEAZLZu3bpw/stwDgAoHgoRAJgCJjV4eajuzZi01atXh/Nzw3kjTPUpgwGg6ChEAACZdXZ2hvMTwzkAoHiYNQsA0FbKPWV6QQCgBVCIAAAy6+/vV39/f3p+b7/6703PAQDFw6VZAIDMenp6JEldXV3J+W1xPic5BwAUD4UIACCzpUuXhvOTwzkAoHgoRAAAma1cuTKcnxnOAQDFQyECAEAGe24/q+Z9tx9XeUxt+087/eZJtAgAWgOD1QEAme3YsUM7duxIz3fv0I7d6TkAoHjoEQGAKWL60gXNbsKkzZ49W5JULidPmzt7VZy30bS6Rww8rd0DQxN6TOmq7ZLGX8ByIr0sANCqKEQAAJnNmDEjnE8L5wCA4qEQAYAEzfxGeqLfsk9Eqa/UkONu2bIlnF8azgEAxcMYEQAAAAC5o0cEAALaddaidhqr0YoqY0XSVGbXStpvvPElANAqKEQAAJnNnz9fkrR+/frk/No4vzA5b1UTnmBgwTUNaQcAtCIKEQBAZps2bQrnD4fzoqi1N6Oyzkj1/uP1ogBAq6EQAYAM+HAYGRoaCudLwnmrmeiEAq08NTMANAqFCAAgs3nz5oXzI8I5AKB4KEQAoA4YQAwAwMQwfS8AILMVK1ZoxYoV6fmGFVqxIT0HABQPhQgAILPe3l719vam5xt71bsxPQcAFA+XZgEAMlu2bFk4PzWcI9me28/ad7uytkhlRq3JOFLSnoei2+26Rg6A1kEhAgDIbPny5eH8tHAOACgeChEAAKaYpN6KylTRWSZGGBkZ0ZEPfWDSjweAeqIQAYAGKdLaEcPDw5LSp/Ed3hXnTOMLAIhRiABoadXX0DcCCxbWZsGCBZKkcrmcnK+O857kHABQPBQiANBgE12FuxXNnTs3nB8ezotior1kSe+dbMXxIfsGvQNAs1GIAGgLWWYACl17z/f3tVm/fn04vzCcN1Kpr7TvNj0yADB1UIgAANBgE+0VS+o5yTJIXeIyQwBTD4UIAKAtVfd+VPeKAACmBgoRACiA137/tQ09fkdHhyRpy5YtyfmVcX5pcg4AKB4KEQBAZjt37gzne8I58teoGedYsR1ArShEAKBAGjVYe/PmzeH8knAOACgeChEAQGYzZ84M59PDOfJz1INfn/RjQwPmG72mD4D2QyECoC0wIxAQVr74KI2MjGjWrFkTfiz/fQFoBAoRAA3XLt+UTnQxuiLp7u6WJK1cuTI5vzXOz0zOAQDFQyECoG1kXWcBkzcwMCApvRAZuC/OKUQAADEKEQC5adRsOnlfNjLRxemKoK+vL5yfEc7RPhrZA8qMXEB7oRABAGTW1dUVzueEc7SGUNG//bgcGwKgLVCIAACAzGqZjWuyl0+2yzgzAKNRiAAAMhscHJQkdXZ2Judb4/zE5BzJJjpBQqMuG8w6/opZtwAkoRABIKn53zjyQaW1LVmyRJJULicvmLhkbZw3aEFFFAPjT4D2QiECoLCYjrd+Fi5cGM5PCOcYbaI9G5X38njv6VMyPg8A1BOFCIBRJvutYE09GjXswxS8rWnNmjXhfFE4B0Ia2VvR7N5goMgoRAC0jcn2cPCtMFpdre/hysrqzeoNbOYlmON9ycFlX0D+citEzOwsSSslHSBplbt/dkx+kKRvSnq1pEclLXb3X+XVPqAVTPYfyiMl7Xmotn2zflCgRwNoHVNlMHwe0v7flse0wxQ5QLJcChEzO0DSVyS9SdI2SXeY2Vp3/2nVbu+S9Li7v9zMLpDUK2lxHu1DMdEdP/XRw9E6SqWSpPTB6qW+OGewekubbOHSzC8oxvtypdWnHZ7oc9T6xRQFDvJg7t74JzE7TdLl7r4wvr9cktx9RdU+6+J9NpjZgZJ2SnqpVzXwiSeeaHxjAQAAANTVYYcdZmO3PSen5z5KUnX9vS3elriPu++V9ISkF+fSOgAAAAC5yqsQAQAAAIB98hqsvl3SMVX3j463Je2zLb406zBFg9b3SerSAQAAANB68uoRuUPSLDM7wcyeJ+kCSWvH7LNW0tL49vmSbvc8BrAAAAAAyF0uPSLuvtfM3i9pnaLpe7/h7veb2acl3enuayV9XdLVZvaApMcUFSsAAAAA2lAus2ZhajCzD0v6vKLZyB5pdnuQLzO7QtI5kp6R9AtJF7s7c6kWwHjrOKG9mdkxitbpOkKSS/qau69sbqvQDPFyCndK2u7uZze7PcifmZUkrZJ0sqL/H7zT3Tc0qz0MVi+I+B+iv5b062a3BU1zi6ST3f1Vkn4uaXmT24McVK3j1CnpFZKWmNkrmtsq5GyvpA+7+ysk/bmk9/EeKKxuSZub3Qg01UpJN7t7h6S5avL7gUKkOL4o6WOKql8UkLv/Vzw1tiT9WNGkEWh/r5P0gLtvdfdnJF0naVGT24QcufsOd787vv2kog8ezVthEE1hZkdLeouib8NRQGZ2mKS/VDQcQu7+TLOvjKAQKQAzW6SoG3ZTs9uCKeOdkgab3QjkopZ1nFAQZna8pFMkbWxuS9AEfYq+kPxTsxuCpjlB0m8lXWVmPzGzVWY2rZkNymv6XjSYmd0qaUZC9PeSLlN0WRbaXOh94O43xPv8vaJLNa7Js20AmsvMpkv6D0k97v67ZrcH+TGzsyU97O53mdmCZrcHTXOgpP8l6QPuvtHMVkr6uKRPNLNBaAPufmbSdjObo6gC3mRmUnQ5zt1m9jp335ljE5GDtPdBhZl1STpb0hlMj10YtazjhDZnZs9VVIRc4+7faXZ7kLs3SDrXzN4s6WBJLzCzb7n7/2lyu5CvbZK2uXulR/R6RYVI0zBrVsGY2a8kvYZZs4onnjnpC5Lmu/tvm90e5CNeIPbnks5QVIDcIelCd7+/qQ1Dbiz6FmpA0mPu3tPs9qC54h6RjzBrVjGZ2Q8lXeLuPzOzyyVNc/ePNqs99IgAxfFlSQdJuiXuHfuxu7+3uU1Co6Wt49TkZiFfb5B0kaR7zWw43naZu9/UxDYBaI4PSLomXmB8q6SLm9kYekQAAAAA5I5ZswAAAADkjkIEAAAAQO4oRAAAAADkjkIEAAAAQO4oRAAAAADkjkIEQGGZ2eVm9q349rFmttvMDqjhcV81s9SVaM3Mzezl9WwfsjGzz5rZo2a2Lb5/vplti//mcxr83EeY2RYzO6iRz9NMFrnTzDqa3RYArYNCBEBLM7NfmVlwRflauPuv3X26u/9PDfu+193/MetzTpaZvT3+AL3bzJ4ysz9V3d/dgOc7MC6ujq/3sWt8/s+YWf84+2yLz8Xuqp++ODtB0gclneTuR8cP+RdJ74n/5vdOsl21npfLJK1y9z/Ej/uRmXWZ2dLA37E83uuqasfXzeyd8e2jzOwqM9tpZr8zs81m9ikze37V/s8xswfN7J4xx/lZ1XP8j5k9XXX/Y2Z2Sbx995ifwz1aC+ALkv5hMucSQDFRiABAi3H3a+IP0NMldUr6TeV+vG2UeHX1IuisPg9Vq4gfJ+lhd39Eij6ISzpGUsMXdowLgIskXTM2c/eBqr/ZOZJ+XdX2UtWuaa+rsmr6WZIGzewlkjYoWqz4VHd/QZy9VNKJVcf7K0kvktRhZqdUteekqvZskPTequf8XLzbD8e0Zbq7Pxxn35X012Z2eIZTBqBAKEQAtI34W+YfmdnnzexxM/ulmXVW5SeY2Xoze9LMbpH0kqrs+Pjb7QPNbLGZ3Tnm2B8ys7Xx7X4z+0xV9lEz22Fmv6l8M12VDZnZJWPbWHV/pZk9FH97fZeZvbFO52Jb3K57Je2Jtx1tZv9pZr+Nz837qvY/zcx+bGbl+LV8ycyeG8c/iH/fH38D/jdmdmbcG7U8Pt5vzOwcMzvbzEbM7DEz+1jV8Z9jZpeZ2S/M7BEzu87MXhhnL4/P/Tvidv/WzD4eZ2dL+pikSi/QXRM8D2dJGpRUufTuKkm/k2Tx6/lZDefmQDP7RNz231l0CdKRSecloQmnKSqCdkyk3RNwiqRd8fE/IulRSe9w9wclyd0fdPf3u3t10bVU0nck3Rzfrgt3/72kYUlvqtcxAbQ3ChEA7eYXz2uDAAAFwElEQVRUST9TVGR8TtLX42+NJelaSXfF2T8q/UPY9ySdZGazqrZdGD9+lPiD7kcUffiaJWmil4ndIWmeom+or5X0bTM7eILHSHOBoh6TUtwLcGP8fEfF7f2omZ0R77tXUreic/MGRd+kvyfO/jL+/cr4G/D/iO8frejfkSMVnc+vx895iqQFkj5tZsfG+35I0lviYx0tabekL41p7+slvVzSQkn/YGaz3P1GRX/HSi/QqydyAtz9Zo3ubbhYUqW34ZXuflIN5+ajks6Pz0lJ0iWSng6cl2pzFL0fG+XNkr4f3z5T0nfiy6QSmdl0SW9V1ENzjaQL69xjtlnS3DoeD0AboxAB0G4edPcr47EeA5JmSjoi/kD8WkmfcPc/uPsPFBUc+4m/2b1B0hJJiguSDklrE3Z/m6Sr3P0+d98j6fKJNNbdv+Xuj7r7Xnf/F0kHSTppIscIWOnu29z9KUXfzL/A3f/Z3Z9x9wf0bOEgd7/D3TfG7dgq6WuS5o9z/Kclfdbd/yjpOkWXAH3R3Xe7+z2KPoC/Kt73vZIuc/ft7v60orEEfxsXARWXu/vT7n63osumJvqB9sa4R6fyc3GNjwueG0WFx2XuPuLuf3L3YXd/rMZjlyQ9ObGXsZ/Q63qLpJvi2y+WNF7Py/mKisDbFL2fn6+oWK3VX4xpy9gi60k9W+gBQFBRrhsGUBw7Kzfc/fdxZ8h0Rd/0Px4XCxUPKhorkORaRQOaP62oN+S7cYEy1pGKelmqj1kzM/uIpHfFx3FJL1DVJWMZPVR1+zhFlyeVq7YdIGkobkeHotf7akmHKPr3YeM4x3+kanD/U/HvXVX5U4rOvSQdK+l7ZvanMcfYN57A3XdWbf991WNrdba7D03wMdI450bRe+QXkziuJD0u6dBJPrYi8XWZ2YsVjf2o/J0eVVR4hyyVtCb+uz1lZv8Zb0ssyhP8yN0XBPJDJZUDOQDsQ48IgKLYIemFZjatatuxaTtLukXSS81snqKekf0uy6o6bnUxM/aYexR9sK+YUbkRjwf5mKJelRfGA5SfUDR+oR6qL9F5SNKIu5eqfg5193Pi/P9Kuk/Sy+NBzp+sakfqpT4TsE3Sm8Y8/8Fjio9aXkcjjHduHpL0skm26x5Jf1avho5xlqRb3b1S3N0q6X9XXYo4ipkdp6iXq8uiWbV2SjpP0jmV8Tp1MFvSpjodC0CboxABUAjx4N07FY09eJ6Z/YWisQNp+/9R0rclXaFo/MYtKbv+u6IPdq8ws0MkfWpMPizprWZ2iEVri7yrKjtU0diM30o60Mw+qahHpBE2SHrGzD5sZgeb2QFmNsfMKmMuDlVUBO0xs9l6dnyI4m/PH9XomZcm6quS/rkyZsTMDjezc2t87C5Jx6d9wK6D8c7NKkmfMbOXWWSemb2oxvOyQVFBOyOwz2RVjw+RpM8r6k27quo8H23RhAivlPQOST9VdOnfvPjnJEW9iBcoI4tmCJunqCACgHFRiAAokgsVDWZ/TFHB8M1x9r9W0QDgb7v73qQd3H1QUp+k2yU9EP+u9kVJzyj6MD2g0dO4rlM0c9HPFV3S9bRGX05VN3H73yzpdZJ+JekRRb0glcLnw4ou0Xky3r5mzCE+JenaeFzAWyfRhC8oeq23mdmTkv5b0ZidWqyR9DxJj5nZ/wvsN2ij17f4di0Hr+HcXKFoatrbFM249TVJlQkFguclXjvkaklvr6UtKfZ7XfHYmjcpeg9VnusRReNdJOmO+DzfEr+erYoKka+4+86qnx3xa6119qw32v7riFSmAD5P0i3uvit0AACosMDkGgAAICMzO0LReJN5cWFSj2O+XtLn3f319TheVnFv1R2SLnL3zc1uD4DWQCECAECLMbM/l3SYu68bd2cAmKIoRAAAAADkjjEiAAAAAHJHIQIAAAAgdxQiAAAAAHJHIQIAAAAgdxQiAAAAAHJHIQIAAAAgd/8fLj+MjYQbQqkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "groups = learner_r._classes\n",
    "\n",
    "alpha = 1\n",
    "linewidth = 2\n",
    "bins = 30\n",
    "for group,idx in sorted(groups.items(), key=lambda x: x[1]):\n",
    "    plt.figure(figsize=(12,8))\n",
    "    plt.hist(cate_t[:,idx], alpha=alpha, bins=bins, label='T Learner ({})'.format(group),\n",
    "             histtype='step', linewidth=linewidth, density=True)\n",
    "    plt.hist(cate_x[:,idx], alpha=alpha, bins=bins, label='X Learner ({})'.format(group),\n",
    "             histtype='step', linewidth=linewidth, density=True)\n",
    "    plt.hist(cate_r[:,idx], alpha=alpha, bins=bins, label='R Learner ({})'.format(group),\n",
    "             histtype='step', linewidth=linewidth, density=True)\n",
    "    plt.hist(tau, alpha=alpha, bins=bins, label='Actual ATE distr',\n",
    "             histtype='step', linewidth=linewidth, color='green', density=True)\n",
    "    plt.vlines(cate_s[0,idx], 0, plt.axes().get_ylim()[1], label='S Learner ({})'.format(group),\n",
    "               linestyles='dotted', linewidth=linewidth)\n",
    "    plt.vlines(tau.mean(), 0, plt.axes().get_ylim()[1], label='Actual ATE',\n",
    "               linestyles='dotted', linewidth=linewidth, color='green')\n",
    "    \n",
    "    plt.title('Distribution of CATE Predictions for {}'.format(group))\n",
    "    plt.xlabel('Individual Treatment Effect (ITE/CATE)')\n",
    "    plt.ylabel('# of Samples')\n",
    "    _=plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Multiple Treatment Case"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate synthetic data\n",
    "Note: we randomize the assignment of treatment flag AFTER the synthetic data generation process, so it doesn't make sense to measure accuracy metrics here. Next steps would be to include multi-treatment in the DGP itself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:59.357345Z",
     "start_time": "2020-04-14T19:04:59.307042Z"
    }
   },
   "outputs": [],
   "source": [
    "# Generate synthetic data using mode 1\n",
    "y, X, treatment, tau, b, e = synthetic_data(mode=1, n=10000, p=8, sigma=1.0)\n",
    "\n",
    "treatment = np.array([('treatment_a' if np.random.random() > 0.2 else 'treatment_b') \n",
    "                      if val==1 else 'control' for val in treatment])\n",
    "\n",
    "e = {group: e for group in np.unique(treatment)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:04:59.412822Z",
     "start_time": "2020-04-14T19:04:59.359396Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "control        4768\n",
       "treatment_a    4146\n",
       "treatment_b    1086\n",
       "dtype: int64"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(treatment).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "## S-Learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:05:01.278019Z",
     "start_time": "2020-04-14T19:04:59.415228Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.6339\n",
      "INFO:causalml:    RMSE (Treatment):     0.6447\n",
      "INFO:causalml:   sMAPE   (Control):     0.6148\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3498\n",
      "INFO:causalml:    Gini   (Control):     0.8528\n",
      "INFO:causalml:    Gini (Treatment):     0.8492\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.5584\n",
      "INFO:causalml:    RMSE (Treatment):     0.4771\n",
      "INFO:causalml:   sMAPE   (Control):     0.5699\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2768\n",
      "INFO:causalml:    Gini   (Control):     0.8921\n",
      "INFO:causalml:    Gini (Treatment):     0.9227\n"
     ]
    }
   ],
   "source": [
    "learner_s = BaseSRegressor(XGBRegressor(), control_name='control')\n",
    "ate_s = learner_s.estimate_ate(X=X, treatment=treatment, y=y, return_ci=False, bootstrap_ci=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:05:01.320962Z",
     "start_time": "2020-04-14T19:05:01.279909Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.58349553, 0.58778215])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ate_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:05:01.368038Z",
     "start_time": "2020-04-14T19:05:01.323307Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'treatment_a': 0, 'treatment_b': 1}"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner_s._classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:05:03.211605Z",
     "start_time": "2020-04-14T19:05:01.370785Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.6339\n",
      "INFO:causalml:    RMSE (Treatment):     0.6447\n",
      "INFO:causalml:   sMAPE   (Control):     0.6148\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3498\n",
      "INFO:causalml:    Gini   (Control):     0.8528\n",
      "INFO:causalml:    Gini (Treatment):     0.8492\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.5584\n",
      "INFO:causalml:    RMSE (Treatment):     0.4771\n",
      "INFO:causalml:   sMAPE   (Control):     0.5699\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2768\n",
      "INFO:causalml:    Gini   (Control):     0.8921\n",
      "INFO:causalml:    Gini (Treatment):     0.9227\n"
     ]
    }
   ],
   "source": [
    "alpha = 0.05\n",
    "learner_s = BaseSRegressor(XGBRegressor(), ate_alpha=alpha, control_name='control')\n",
    "ate_s, ate_s_lb, ate_s_ub = learner_s.estimate_ate(X=X, treatment=treatment, y=y, return_ci=True,\n",
    "                                                   bootstrap_ci=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:05:03.255641Z",
     "start_time": "2020-04-14T19:05:03.213558Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.5555693 , 0.55278018],\n",
       "       [0.58349553, 0.58778215],\n",
       "       [0.61142176, 0.62278413]])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_s_lb, ate_s, ate_s_ub))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Boostrap Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:06:45.403405Z",
     "start_time": "2020-04-14T19:05:03.258090Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.6339\n",
      "INFO:causalml:    RMSE (Treatment):     0.6447\n",
      "INFO:causalml:   sMAPE   (Control):     0.6148\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3498\n",
      "INFO:causalml:    Gini   (Control):     0.8528\n",
      "INFO:causalml:    Gini (Treatment):     0.8492\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.5584\n",
      "INFO:causalml:    RMSE (Treatment):     0.4771\n",
      "INFO:causalml:   sMAPE   (Control):     0.5699\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2768\n",
      "INFO:causalml:    Gini   (Control):     0.8921\n",
      "INFO:causalml:    Gini (Treatment):     0.9227\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [01:40<00:00,  1.00s/it]\n"
     ]
    }
   ],
   "source": [
    "ate_s_b, ate_s_lb_b, ate_s_ub_b = learner_s.estimate_ate(X=X, treatment=treatment, y=y, return_ci=True,\n",
    "                                                         bootstrap_ci=True, n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:06:45.442749Z",
     "start_time": "2020-04-14T19:06:45.405407Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.52550035, 0.52550035],\n",
       "       [0.58349553, 0.58778215],\n",
       "       [0.64944596, 0.64944596]])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_s_lb_b, ate_s_b, ate_s_ub_b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:06:47.149107Z",
     "start_time": "2020-04-14T19:06:45.444724Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.6339\n",
      "INFO:causalml:    RMSE (Treatment):     0.6447\n",
      "INFO:causalml:   sMAPE   (Control):     0.6148\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3498\n",
      "INFO:causalml:    Gini   (Control):     0.8528\n",
      "INFO:causalml:    Gini (Treatment):     0.8492\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.5584\n",
      "INFO:causalml:    RMSE (Treatment):     0.4771\n",
      "INFO:causalml:   sMAPE   (Control):     0.5699\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2768\n",
      "INFO:causalml:    Gini   (Control):     0.8921\n",
      "INFO:causalml:    Gini (Treatment):     0.9227\n"
     ]
    }
   ],
   "source": [
    "learner_s = BaseSRegressor(XGBRegressor(), control_name='control')\n",
    "cate_s = learner_s.fit_predict(X=X, treatment=treatment, y=y, return_ci=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:06:47.187393Z",
     "start_time": "2020-04-14T19:06:47.150866Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.91381967,  0.82956386],\n",
       "       [-0.17692167, -0.15709245],\n",
       "       [ 0.90877771,  0.92332006],\n",
       "       ...,\n",
       "       [ 0.86159408,  0.53687155],\n",
       "       [ 0.66541922,  0.78590739],\n",
       "       [ 1.05691028,  1.03345728]])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:07:52.420017Z",
     "start_time": "2020-04-14T19:06:47.189370Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.6339\n",
      "INFO:causalml:    RMSE (Treatment):     0.6447\n",
      "INFO:causalml:   sMAPE   (Control):     0.6148\n",
      "INFO:causalml:   sMAPE (Treatment):     0.3498\n",
      "INFO:causalml:    Gini   (Control):     0.8528\n",
      "INFO:causalml:    Gini (Treatment):     0.8492\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.5584\n",
      "INFO:causalml:    RMSE (Treatment):     0.4771\n",
      "INFO:causalml:   sMAPE   (Control):     0.5699\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2768\n",
      "INFO:causalml:    Gini   (Control):     0.8921\n",
      "INFO:causalml:    Gini (Treatment):     0.9227\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [01:03<00:00,  1.58it/s]\n"
     ]
    }
   ],
   "source": [
    "alpha = 0.05\n",
    "learner_s = BaseSRegressor(XGBRegressor(), ate_alpha=alpha, control_name='control')\n",
    "cate_s, cate_s_lb, cate_s_ub = learner_s.fit_predict(X=X, treatment=treatment, y=y, return_ci=True,\n",
    "                               n_bootstraps=100, bootstrap_size=3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:07:52.463305Z",
     "start_time": "2020-04-14T19:07:52.422192Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.91381967,  0.82956386],\n",
       "       [-0.17692167, -0.15709245],\n",
       "       [ 0.90877771,  0.92332006],\n",
       "       ...,\n",
       "       [ 0.86159408,  0.53687155],\n",
       "       [ 0.66541922,  0.78590739],\n",
       "       [ 1.05691028,  1.03345728]])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:07:52.503242Z",
     "start_time": "2020-04-14T19:07:52.465394Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.23816384, -0.32713253],\n",
       "       [-0.44141183, -0.42676411],\n",
       "       [-0.00206863, -0.43860602],\n",
       "       ...,\n",
       "       [ 0.29240462, -0.16563866],\n",
       "       [-0.01797467, -0.10772878],\n",
       "       [-0.51486325, -0.31691882]])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_s_lb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:07:52.543787Z",
     "start_time": "2020-04-14T19:07:52.505112Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.40557503, 1.1807412 ],\n",
       "       [1.06860972, 1.55298753],\n",
       "       [1.38529261, 1.6596471 ],\n",
       "       ...,\n",
       "       [1.56729684, 1.47052228],\n",
       "       [1.16166003, 1.1144281 ],\n",
       "       [1.68127107, 1.58984778]])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_s_ub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "## T-Learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:07:54.253387Z",
     "start_time": "2020-04-14T19:07:52.545793Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n"
     ]
    }
   ],
   "source": [
    "learner_t = BaseTRegressor(XGBRegressor(), control_name='control')\n",
    "ate_t, ate_t_lb, ate_t_ub = learner_t.estimate_ate(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:07:54.292831Z",
     "start_time": "2020-04-14T19:07:54.255519Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.53107041, 0.5296616 ],\n",
       "       [0.55739303, 0.55794811],\n",
       "       [0.58371565, 0.58623463]])"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_t_lb, ate_t, ate_t_ub))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Boostrap Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:09:28.986981Z",
     "start_time": "2020-04-14T19:07:54.294826Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [01:32<00:00,  1.08it/s]\n"
     ]
    }
   ],
   "source": [
    "ate_t_b, ate_t_lb_b, ate_t_ub_b = learner_t.estimate_ate(X=X, treatment=treatment, y=y, bootstrap_ci=True,\n",
    "                                                   n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:09:29.025336Z",
     "start_time": "2020-04-14T19:09:28.988777Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.51777538, 0.51777538],\n",
       "       [0.55739303, 0.55794811],\n",
       "       [0.67471492, 0.67471492]])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_t_lb_b, ate_t_b, ate_t_ub_b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:09:30.687586Z",
     "start_time": "2020-04-14T19:09:29.027317Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n"
     ]
    }
   ],
   "source": [
    "learner_t = BaseTRegressor(XGBRegressor(), control_name='control')\n",
    "cate_t = learner_t.fit_predict(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:09:30.724632Z",
     "start_time": "2020-04-14T19:09:30.689302Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.47525787, -0.06651461],\n",
       "       [ 1.26169336,  1.14718354],\n",
       "       [ 1.68760026,  0.75878632],\n",
       "       ...,\n",
       "       [ 0.37292147,  0.20537615],\n",
       "       [ 0.84290075,  0.80045319],\n",
       "       [ 1.64227223,  1.91352534]])"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:10:38.696792Z",
     "start_time": "2020-04-14T19:09:30.726511Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n"
     ]
    }
   ],
   "source": [
    "learner_t = BaseTRegressor(XGBRegressor(), control_name='control')\n",
    "cate_t, cate_t_lb, cate_t_ub = learner_t.fit_predict(X=X, treatment=treatment, y=y, return_ci=True, n_bootstraps=100,\n",
    "                                                    bootstrap_size=3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:10:38.738058Z",
     "start_time": "2020-04-14T19:10:38.698876Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.47525787, -0.06651461],\n",
       "       [ 1.26169336,  1.14718354],\n",
       "       [ 1.68760026,  0.75878632],\n",
       "       ...,\n",
       "       [ 0.37292147,  0.20537615],\n",
       "       [ 0.84290075,  0.80045319],\n",
       "       [ 1.64227223,  1.91352534]])"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:10:38.778042Z",
     "start_time": "2020-04-14T19:10:38.739946Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.18706408, -0.84940575],\n",
       "       [-1.01419897, -0.7311732 ],\n",
       "       [-0.0427315 , -0.16378173],\n",
       "       ...,\n",
       "       [-0.39076423, -0.16869925],\n",
       "       [-0.17401927, -0.19503389],\n",
       "       [-0.61903974, -1.15808628]])"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_t_lb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:10:38.817236Z",
     "start_time": "2020-04-14T19:10:38.780066Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2.47563672, 1.69891493],\n",
       "       [2.04089584, 1.76605188],\n",
       "       [2.3567108 , 2.40833322],\n",
       "       ...,\n",
       "       [2.17926003, 2.26919731],\n",
       "       [2.15714553, 1.91076722],\n",
       "       [2.27031788, 2.03901908]])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_t_ub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## X-Learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:10:42.153573Z",
     "start_time": "2020-04-14T19:10:38.819233Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n"
     ]
    }
   ],
   "source": [
    "learner_x = BaseXRegressor(XGBRegressor(), control_name='control')\n",
    "ate_x, ate_x_lb, ate_x_ub = learner_x.estimate_ate(X=X, treatment=treatment, y=y, p=e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:10:42.191367Z",
     "start_time": "2020-04-14T19:10:42.155488Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.49573269, 0.54002602],\n",
       "       [0.51860246, 0.56163457],\n",
       "       [0.54147223, 0.58324311]])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_x_lb, ate_x, ate_x_ub))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:10:46.431322Z",
     "start_time": "2020-04-14T19:10:42.193271Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n"
     ]
    }
   ],
   "source": [
    "ate_x_no_p, ate_x_lb_no_p, ate_x_ub_no_p = learner_x.estimate_ate(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:10:46.467980Z",
     "start_time": "2020-04-14T19:10:46.433128Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.50418298, 0.56976992],\n",
       "       [0.52706595, 0.59243233],\n",
       "       [0.54994892, 0.61509475]])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_x_lb_no_p, ate_x_no_p, ate_x_ub_no_p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Boostrap Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:13:45.310480Z",
     "start_time": "2020-04-14T19:10:46.469940Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [02:55<00:00,  1.75s/it]\n"
     ]
    }
   ],
   "source": [
    "ate_x_b, ate_x_lb_b, ate_x_ub_b = learner_x.estimate_ate(X=X, treatment=treatment, y=y, p=e, bootstrap_ci=True,\n",
    "                                                   n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:13:45.355233Z",
     "start_time": "2020-04-14T19:13:45.312425Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.49600789, 0.49600789],\n",
       "       [0.51860246, 0.56163457],\n",
       "       [0.63696386, 0.63696386]])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_x_lb_b, ate_x_b, ate_x_ub_b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:16:44.130037Z",
     "start_time": "2020-04-14T19:13:45.357393Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [02:54<00:00,  1.74s/it]\n"
     ]
    }
   ],
   "source": [
    "ate_x_b_no_p, ate_x_lb_b_no_p, ate_x_ub_b_no_p = learner_x.estimate_ate(X=X, treatment=treatment, y=y, bootstrap_ci=True,\n",
    "                                                   n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:16:44.187331Z",
     "start_time": "2020-04-14T19:16:44.132067Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.50100288, 0.50100288],\n",
       "       [0.52706414, 0.59242806],\n",
       "       [0.66020792, 0.66020792]])"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_x_lb_b_no_p, ate_x_b_no_p, ate_x_ub_b_no_p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:16:47.515109Z",
     "start_time": "2020-04-14T19:16:44.189448Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n"
     ]
    }
   ],
   "source": [
    "learner_x = BaseXRegressor(XGBRegressor(), control_name='control')\n",
    "cate_x = learner_x.fit_predict(X=X, treatment=treatment, y=y, p=e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:16:47.556487Z",
     "start_time": "2020-04-14T19:16:47.516863Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.57149441,  0.10240081],\n",
       "       [-0.43192272,  1.48913118],\n",
       "       [ 1.13622262,  0.65923928],\n",
       "       ...,\n",
       "       [ 0.44651704, -0.23119723],\n",
       "       [ 0.93875551,  0.77003003],\n",
       "       [ 0.96697381,  0.99990004]])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:16:51.907370Z",
     "start_time": "2020-04-14T19:16:47.558866Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n"
     ]
    }
   ],
   "source": [
    "cate_x_no_p = learner_x.fit_predict(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:16:51.951219Z",
     "start_time": "2020-04-14T19:16:51.909187Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.62959351, -0.00493521],\n",
       "       [-0.48863166,  1.54109948],\n",
       "       [ 1.17988308,  1.26200671],\n",
       "       ...,\n",
       "       [ 0.41320951,  0.73251634],\n",
       "       [ 0.91104634,  0.82359481],\n",
       "       [ 1.08867931,  1.44193089]])"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_no_p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:17:46.988230Z",
     "start_time": "2020-04-14T19:16:51.953440Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [00:51<00:00,  1.94it/s]\n"
     ]
    }
   ],
   "source": [
    "learner_x = BaseXRegressor(XGBRegressor(), control_name='control')\n",
    "cate_x, cate_x_lb, cate_x_ub = learner_x.fit_predict(X=X, treatment=treatment, y=y, p=e, return_ci=True,\n",
    "                                                     n_bootstraps=100, bootstrap_size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:17:47.035158Z",
     "start_time": "2020-04-14T19:17:46.990429Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'treatment_a': 0, 'treatment_b': 1}"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner_x._classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:17:47.080571Z",
     "start_time": "2020-04-14T19:17:47.037415Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.57149441,  0.10240081],\n",
       "       [-0.43192272,  1.48913118],\n",
       "       [ 1.13622262,  0.65923928],\n",
       "       ...,\n",
       "       [ 0.44651704, -0.23119723],\n",
       "       [ 0.93875551,  0.77003003],\n",
       "       [ 0.96697381,  0.99990004]])"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:17:47.125458Z",
     "start_time": "2020-04-14T19:17:47.082758Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.23574115, -0.21029023],\n",
       "       [-0.95699419, -1.05203708],\n",
       "       [-0.49402807, -0.48280283],\n",
       "       ...,\n",
       "       [-0.12162789, -0.26408791],\n",
       "       [-0.52562958, -0.19338615],\n",
       "       [-0.40858565, -0.88119588]])"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_lb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:17:47.171213Z",
     "start_time": "2020-04-14T19:17:47.127785Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.79950407, 2.11258332],\n",
       "       [1.45309225, 1.48831446],\n",
       "       [1.75564219, 2.03222137],\n",
       "       ...,\n",
       "       [2.15191078, 2.30032378],\n",
       "       [1.65228261, 1.40411322],\n",
       "       [1.74815254, 1.68257617]])"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_ub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:18:43.066112Z",
     "start_time": "2020-04-14T19:17:47.173533Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Error metrics for group treatment_a\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.4669\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.2675\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9297\n",
      "INFO:causalml:Error metrics for group treatment_b\n",
      "INFO:causalml:    RMSE   (Control):     0.4743\n",
      "INFO:causalml:    RMSE (Treatment):     0.0747\n",
      "INFO:causalml:   sMAPE   (Control):     0.5062\n",
      "INFO:causalml:   sMAPE (Treatment):     0.0568\n",
      "INFO:causalml:    Gini   (Control):     0.9280\n",
      "INFO:causalml:    Gini (Treatment):     0.9984\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [00:51<00:00,  1.94it/s]\n"
     ]
    }
   ],
   "source": [
    "cate_x_no_p, cate_x_lb_no_p, cate_x_ub_no_p = learner_x.fit_predict(X=X, treatment=treatment, y=y, return_ci=True,\n",
    "                                                     n_bootstraps=100, bootstrap_size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:18:43.114297Z",
     "start_time": "2020-04-14T19:18:43.068442Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'treatment_a': 0, 'treatment_b': 1}"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner_x._classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:18:43.159279Z",
     "start_time": "2020-04-14T19:18:43.116452Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.6294132 , -0.00492528],\n",
       "       [-0.48876998,  1.54111376],\n",
       "       [ 1.17989094,  1.2620318 ],\n",
       "       ...,\n",
       "       [ 0.41319463,  0.73237091],\n",
       "       [ 0.9108665 ,  0.82359564],\n",
       "       [ 1.08868219,  1.441931  ]])"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_no_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:18:43.206463Z",
     "start_time": "2020-04-14T19:18:43.162141Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.10073893, -0.38800051],\n",
       "       [-0.81971717, -0.8298923 ],\n",
       "       [-0.18606629, -0.32586878],\n",
       "       ...,\n",
       "       [ 0.18372251, -0.12170252],\n",
       "       [-0.21309623, -0.38600234],\n",
       "       [-0.44863794, -0.39716903]])"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_lb_no_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:18:43.251400Z",
     "start_time": "2020-04-14T19:18:43.208825Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2.00312255, 2.10486085],\n",
       "       [1.59355675, 1.76340695],\n",
       "       [1.77980204, 2.35535097],\n",
       "       ...,\n",
       "       [1.94828429, 1.94720835],\n",
       "       [2.04021647, 1.71337955],\n",
       "       [1.60121219, 1.82820234]])"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_x_ub_no_p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## R-Learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:18:49.522197Z",
     "start_time": "2020-04-14T19:18:43.253881Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:training the treatment effect model for treatment_b with R-loss\n"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "ate_r, ate_r_lb, ate_r_ub = learner_r.estimate_ate(X=X, treatment=treatment, y=y, p=e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:18:49.569287Z",
     "start_time": "2020-04-14T19:18:49.524357Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.52326968, 0.57744164],\n",
       "       [0.52374892, 0.5781462 ],\n",
       "       [0.52422816, 0.57885076]])"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_r_lb, ate_r, ate_r_ub))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:18:54.689767Z",
     "start_time": "2020-04-14T19:18:49.571426Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:training the treatment effect model for treatment_b with R-loss\n"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "ate_r_no_p, ate_r_lb_no_p, ate_r_ub_no_p = learner_r.estimate_ate(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:18:54.730346Z",
     "start_time": "2020-04-14T19:18:54.691652Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.44161159, 0.71836119],\n",
       "       [0.44209269, 0.71904979],\n",
       "       [0.44257378, 0.71973838]])"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_r_lb_no_p, ate_r_no_p, ate_r_ub_no_p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:18:54.779756Z",
     "start_time": "2020-04-14T19:18:54.732335Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'treatment_a': {'all training': LogisticRegressionCV(Cs=array([1.00230524, 2.15608891, 4.63802765, 9.97700064]),\n",
       "                       class_weight=None,\n",
       "                       cv=KFold(n_splits=5, random_state=None, shuffle=True),\n",
       "                       dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "                       l1_ratios=array([0.001     , 0.33366667, 0.66633333, 0.999     ]),\n",
       "                       max_iter=100, multi_class='auto', n_jobs=None,\n",
       "                       penalty='elasticnet', random_state=None, refit=True,\n",
       "                       scoring=None, solver='saga', tol=0.0001, verbose=0)},\n",
       " 'treatment_b': {'all training': LogisticRegressionCV(Cs=array([1.00230524, 2.15608891, 4.63802765, 9.97700064]),\n",
       "                       class_weight=None,\n",
       "                       cv=KFold(n_splits=5, random_state=None, shuffle=True),\n",
       "                       dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "                       l1_ratios=array([0.001     , 0.33366667, 0.66633333, 0.999     ]),\n",
       "                       max_iter=100, multi_class='auto', n_jobs=None,\n",
       "                       penalty='elasticnet', random_state=None, refit=True,\n",
       "                       scoring=None, solver='saga', tol=0.0001, verbose=0)}}"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner_r.propensity_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ATE w/ Boostrap Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:21:17.612601Z",
     "start_time": "2020-04-14T19:18:54.781916Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:training the treatment effect model for treatment_b with R-loss\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [02:19<00:00,  1.39s/it]\n"
     ]
    }
   ],
   "source": [
    "ate_r_b, ate_r_lb_b, ate_r_ub_b = learner_r.estimate_ate(X=X, treatment=treatment, y=y, p=e, bootstrap_ci=True,\n",
    "                                                   n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:21:17.655865Z",
     "start_time": "2020-04-14T19:21:17.614542Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.40326436, 0.40326436],\n",
       "       [0.50620059, 0.5478152 ],\n",
       "       [0.5697328 , 0.5697328 ]])"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_r_lb_b, ate_r_b, ate_r_ub_b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:23:41.531458Z",
     "start_time": "2020-04-14T19:21:17.657918Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:training the treatment effect model for treatment_b with R-loss\n",
      "INFO:causalml:Bootstrap Confidence Intervals for ATE\n",
      "100%|██████████| 100/100 [02:19<00:00,  1.39s/it]\n"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "ate_r_b_no_p, ate_r_lb_b_no_p, ate_r_ub_b_no_p = learner_r.estimate_ate(X=X, treatment=treatment, y=y, bootstrap_ci=True,\n",
    "                                                   n_bootstraps=100, bootstrap_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:23:41.578488Z",
     "start_time": "2020-04-14T19:23:41.533496Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.45994051, 0.45994051],\n",
       "       [0.44481491, 0.66323246],\n",
       "       [0.68981572, 0.68981572]])"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((ate_r_lb_b_no_p, ate_r_b_no_p, ate_r_ub_b_no_p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:23:44.255819Z",
     "start_time": "2020-04-14T19:23:41.580879Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:training the treatment effect model for treatment_b with R-loss\n"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "cate_r = learner_r.fit_predict(X=X, treatment=treatment, y=y, p=e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:23:44.297265Z",
     "start_time": "2020-04-14T19:23:44.257762Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 5.57098567e-01,  1.77359581e-03],\n",
       "       [ 1.08587885e+00,  2.48472750e-01],\n",
       "       [ 3.34437251e-01,  1.69020355e+00],\n",
       "       ...,\n",
       "       [-9.96065974e-01, -8.98482800e-02],\n",
       "       [ 1.70625651e+00,  9.55640435e-01],\n",
       "       [-1.88456130e+00,  6.50659442e-01]])"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:23:48.815108Z",
     "start_time": "2020-04-14T19:23:44.299436Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:Generating propensity score\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:Calibrating propensity scores.\n",
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:training the treatment effect model for treatment_b with R-loss\n"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "cate_r_no_p = learner_r.fit_predict(X=X, treatment=treatment, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:23:48.859511Z",
     "start_time": "2020-04-14T19:23:48.817196Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.55478877,  0.87992519],\n",
       "       [ 1.10120189,  1.29564619],\n",
       "       [ 0.62448621,  0.41555083],\n",
       "       ...,\n",
       "       [-0.53886592,  0.44593787],\n",
       "       [ 1.25231111,  0.79904991],\n",
       "       [-0.64419305, -0.23014426]])"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r_no_p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATE w/ Confidence Intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:24:29.398563Z",
     "start_time": "2020-04-14T19:23:48.862628Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:training the treatment effect model for treatment_b with R-loss\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "100%|██████████| 100/100 [00:37<00:00,  2.65it/s]\n"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "cate_r, cate_r_lb, cate_r_ub = learner_r.fit_predict(X=X, treatment=treatment, y=y, p=e, return_ci=True,\n",
    "                                                     n_bootstraps=100, bootstrap_size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:24:29.445452Z",
     "start_time": "2020-04-14T19:24:29.400875Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.75007784,  0.67752302],\n",
       "       [ 0.77257723,  0.12910607],\n",
       "       [ 1.08854032,  0.81679094],\n",
       "       ...,\n",
       "       [-0.92310214,  0.645491  ],\n",
       "       [ 0.92478108,  0.79903334],\n",
       "       [-0.48311949,  1.00291944]])"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:24:29.493876Z",
     "start_time": "2020-04-14T19:24:29.447754Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.801657  , -0.48754777],\n",
       "       [-3.05317249, -5.37572038],\n",
       "       [-1.50823961, -1.16822439],\n",
       "       ...,\n",
       "       [-1.27909884, -1.2460175 ],\n",
       "       [-1.42656819, -1.59059022],\n",
       "       [-1.90115855, -2.10247456]])"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r_lb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-14T19:24:29.541179Z",
     "start_time": "2020-04-14T19:24:29.496419Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4.06750882, 3.68516954],\n",
       "       [4.21587243, 4.50271177],\n",
       "       [4.33370841, 3.79358828],\n",
       "       ...,\n",
       "       [3.53610538, 3.48638564],\n",
       "       [3.71832166, 3.48292163],\n",
       "       [5.01262635, 3.27047309]])"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_r_ub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without Propensity Score Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-04-14T18:47:09.698Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:causalml:generating out-of-fold CV outcome estimates\n",
      "INFO:causalml:training the treatment effect model for treatment_a with R-loss\n",
      "INFO:causalml:training the treatment effect model for treatment_b with R-loss\n",
      "INFO:causalml:Bootstrap Confidence Intervals\n",
      "  2%|▏         | 2/100 [00:00<00:36,  2.69it/s]"
     ]
    }
   ],
   "source": [
    "learner_r = BaseRRegressor(XGBRegressor(), control_name='control')\n",
    "cate_r_no_p, cate_r_lb_no_p, cate_r_ub_no_p = learner_r.fit_predict(X=X, treatment=treatment, y=y, p=e, return_ci=True,\n",
    "                                                     n_bootstraps=100, bootstrap_size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-04-14T18:47:09.702Z"
    }
   },
   "outputs": [],
   "source": [
    "cate_r_no_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-04-14T18:47:09.706Z"
    }
   },
   "outputs": [],
   "source": [
    "cate_r_lb_no_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-04-14T18:47:09.710Z"
    }
   },
   "outputs": [],
   "source": [
    "cate_r_ub_no_p"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "174px",
    "width": "252px"
   },
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "203px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}