GiliGiliEYE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "from sklearn.cross_validation import train_test_split  \n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import KFold\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from dateutil.parser import parse\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def creatDtype():\n",
    "    dtype = {'id':'object',\n",
    "             'label':'int8',\n",
    "             'date':'int64',\n",
    "             'f1':'uint8',\n",
    "             'f2': 'uint8',\n",
    "             'f3': 'uint8',\n",
    "             'f4': 'uint8',\n",
    "             'f5': 'float32',\n",
    "             'ndays':'uint8'\n",
    "             }\n",
    "    for i in range(20,298):\n",
    "        dtype['f'+str(i)] = 'float32'\n",
    "    for i in range(6,20):\n",
    "        dtype['f'+str(i)] = 'uint8'\n",
    "    return dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def appendlist(col,a,b):\n",
    "    for i in range(a,b+1,1):\n",
    "        col.append(\"f\"+str(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data!\n",
      "((994731, 300), (500538, 299))\n"
     ]
    }
   ],
   "source": [
    "print(\"Loading data!\")\n",
    "train = pd.read_csv('atec_anti_fraud_train.csv',dtype=creatDtype())\n",
    "test = pd.read_csv('atec_anti_fraud_test_b.csv',dtype=creatDtype())\n",
    "print(train.shape, test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Concat train and test!\n",
      "(1495269, 300)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "83"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Concat train and test!\")\n",
    "test['label'] = 2\n",
    "len_train = len(train)\n",
    "train = train.append(test)\n",
    "train = train.reset_index(drop=True)\n",
    "print(train.shape)\n",
    "del test\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split data into train test when finish FE!\n",
      "((994731, 300), (500538, 300))\n"
     ]
    }
   ],
   "source": [
    "print(\"Split data into train test when finish FE!\")\n",
    "test = train[len_train:]\n",
    "train = train[:len_train]\n",
    "print(train.shape,test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing remain columns!\n",
      "('pred ', 193)\n"
     ]
    }
   ],
   "source": [
    "print(\"Preparing remain columns!\")\n",
    "pre = [k for k in train.columns if k not in ['id', 'label', 'date', 'ndays']]\n",
    "need_del = []\n",
    "appendlist(need_del,20,27)\n",
    "appendlist(need_del,32,35)\n",
    "appendlist(need_del,48,53)\n",
    "appendlist(need_del,64,71)\n",
    "appendlist(need_del,111,165)\n",
    "appendlist(need_del,211,233)\n",
    "pred = [k for k in pre if  k not in need_del]\n",
    "print(\"pred \", len(pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decline samples!\n",
      "(994731, 301)\n",
      "(643383, 301)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Decline samples!\")\n",
    "train['ndays'] = train['date'].apply(lambda x:(parse(str(x))-parse(str(20170905))).days)\n",
    "print train.shape\n",
    "# train = train[(train.f24 <= 1) | train.f24.isnull()]\n",
    "# train = train[(train.f25 <= 1) | train.f25.isnull()]\n",
    "# train = train[(train.f26 <= 1) | train.f26.isnull()]\n",
    "# train = train[(train.f27 <= 1) | train.f27.isnull()]\n",
    "train = train[(train.f28 <= 1) | train.f28.isnull()]\n",
    "train = train[(train.f29 <= 1) | train.f29.isnull()]\n",
    "train = train[(train.f30 <= 1) | train.f30.isnull()]\n",
    "train = train[(train.f31 <= 1) | train.f31.isnull()]\n",
    "train = train[train['ndays']<=40]\n",
    "print train.shape\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Change -1 to 1!\n",
      " 0    635627\n",
      " 1      4734\n",
      "-1      3022\n",
      "Name: label, dtype: int64\n",
      "0    635627\n",
      "1      7756\n",
      "Name: label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(\"Change -1 to 1!\")\n",
    "print(train.label.value_counts())\n",
    "train['label'] = train['label'].apply(lambda x: 0 if x == 0 else 1)\n",
    "print(train.label.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbparams = {'eta': 0.02, \n",
    "          'tree_method': \"auto\", \n",
    "          'max_depth': 3, \n",
    "          'subsample': 0.9, \n",
    "          'colsample_bytree': 0.1, \n",
    "          'min_child_weight':1,\n",
    "          'objective': 'binary:logistic', \n",
    "          'eval_metric': 'auc', \n",
    "          'random_state': 99,\n",
    "          'silent': True}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K-fold xgb training!\n",
      "('No ', 1)\n",
      "[0]\ttrain-auc:0.866589\tvalid-auc:0.871669\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 100 rounds.\n",
      "[100]\ttrain-auc:0.94697\tvalid-auc:0.949504\n",
      "[200]\ttrain-auc:0.95334\tvalid-auc:0.95529\n",
      "[300]\ttrain-auc:0.960548\tvalid-auc:0.961602\n",
      "[400]\ttrain-auc:0.965653\tvalid-auc:0.966123\n",
      "[500]\ttrain-auc:0.96892\tvalid-auc:0.96898\n",
      "[600]\ttrain-auc:0.971324\tvalid-auc:0.971063\n",
      "[700]\ttrain-auc:0.972901\tvalid-auc:0.972498\n",
      "[800]\ttrain-auc:0.974102\tvalid-auc:0.973476\n",
      "[900]\ttrain-auc:0.97505\tvalid-auc:0.974207\n",
      "[1000]\ttrain-auc:0.975899\tvalid-auc:0.974871\n",
      "[1100]\ttrain-auc:0.976579\tvalid-auc:0.97539\n",
      "[1200]\ttrain-auc:0.97717\tvalid-auc:0.975767\n",
      "[1300]\ttrain-auc:0.977692\tvalid-auc:0.97608\n",
      "[1400]\ttrain-auc:0.978178\tvalid-auc:0.976331\n",
      "[1500]\ttrain-auc:0.978644\tvalid-auc:0.9766\n",
      "[1600]\ttrain-auc:0.979061\tvalid-auc:0.976798\n",
      "[1700]\ttrain-auc:0.979442\tvalid-auc:0.97695\n",
      "[1800]\ttrain-auc:0.979816\tvalid-auc:0.977164\n",
      "[1900]\ttrain-auc:0.980183\tvalid-auc:0.977338\n",
      "[2000]\ttrain-auc:0.980539\tvalid-auc:0.977492\n",
      "[2100]\ttrain-auc:0.980854\tvalid-auc:0.977619\n",
      "[2200]\ttrain-auc:0.981156\tvalid-auc:0.977762\n",
      "[2300]\ttrain-auc:0.981429\tvalid-auc:0.977914\n",
      "[2400]\ttrain-auc:0.981703\tvalid-auc:0.978005\n",
      "[2500]\ttrain-auc:0.981947\tvalid-auc:0.97806\n",
      "[2600]\ttrain-auc:0.982196\tvalid-auc:0.97819\n",
      "[2700]\ttrain-auc:0.982455\tvalid-auc:0.978301\n",
      "[2800]\ttrain-auc:0.982684\tvalid-auc:0.978404\n",
      "[2900]\ttrain-auc:0.982912\tvalid-auc:0.978501\n",
      "[3000]\ttrain-auc:0.983139\tvalid-auc:0.978604\n",
      "[3100]\ttrain-auc:0.983345\tvalid-auc:0.97869\n",
      "[3200]\ttrain-auc:0.98355\tvalid-auc:0.978758\n",
      "[3300]\ttrain-auc:0.983759\tvalid-auc:0.978842\n",
      "[3400]\ttrain-auc:0.983945\tvalid-auc:0.978908\n",
      "[3500]\ttrain-auc:0.984135\tvalid-auc:0.978951\n",
      "[3600]\ttrain-auc:0.984306\tvalid-auc:0.979008\n",
      "[3700]\ttrain-auc:0.984488\tvalid-auc:0.979076\n",
      "[3800]\ttrain-auc:0.984671\tvalid-auc:0.979118\n",
      "[3900]\ttrain-auc:0.984832\tvalid-auc:0.979205\n",
      "[4000]\ttrain-auc:0.984983\tvalid-auc:0.979275\n",
      "[4100]\ttrain-auc:0.985148\tvalid-auc:0.979297\n",
      "[4200]\ttrain-auc:0.985306\tvalid-auc:0.979344\n",
      "[4300]\ttrain-auc:0.98546\tvalid-auc:0.979392\n",
      "[4400]\ttrain-auc:0.985601\tvalid-auc:0.979416\n",
      "[4500]\ttrain-auc:0.985749\tvalid-auc:0.97945\n",
      "[4600]\ttrain-auc:0.985905\tvalid-auc:0.979512\n",
      "[4700]\ttrain-auc:0.986037\tvalid-auc:0.979554\n",
      "[4800]\ttrain-auc:0.986177\tvalid-auc:0.979597\n",
      "[4900]\ttrain-auc:0.986298\tvalid-auc:0.97962\n",
      "[5000]\ttrain-auc:0.986441\tvalid-auc:0.979677\n",
      "[5100]\ttrain-auc:0.986567\tvalid-auc:0.979698\n",
      "[5200]\ttrain-auc:0.986685\tvalid-auc:0.979737\n",
      "[5300]\ttrain-auc:0.986801\tvalid-auc:0.97978\n",
      "[5400]\ttrain-auc:0.986926\tvalid-auc:0.979791\n",
      "[5500]\ttrain-auc:0.987036\tvalid-auc:0.979844\n",
      "[5600]\ttrain-auc:0.987147\tvalid-auc:0.979854\n",
      "[5700]\ttrain-auc:0.987256\tvalid-auc:0.979878\n",
      "[5800]\ttrain-auc:0.987362\tvalid-auc:0.979928\n",
      "[5900]\ttrain-auc:0.987472\tvalid-auc:0.979981\n",
      "[6000]\ttrain-auc:0.987592\tvalid-auc:0.980021\n",
      "[6100]\ttrain-auc:0.987695\tvalid-auc:0.980064\n",
      "[6200]\ttrain-auc:0.987801\tvalid-auc:0.980097\n",
      "[6300]\ttrain-auc:0.987913\tvalid-auc:0.980121\n",
      "[6400]\ttrain-auc:0.988011\tvalid-auc:0.980152\n",
      "[6500]\ttrain-auc:0.988107\tvalid-auc:0.980187\n",
      "[6600]\ttrain-auc:0.988212\tvalid-auc:0.980154\n",
      "Stopping. Best iteration:\n",
      "[6500]\ttrain-auc:0.988107\tvalid-auc:0.980187\n",
      "\n",
      "('No ', 2)\n",
      "[0]\ttrain-auc:0.869215\tvalid-auc:0.8665\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 100 rounds.\n",
      "[100]\ttrain-auc:0.94709\tvalid-auc:0.945876\n",
      "[200]\ttrain-auc:0.954018\tvalid-auc:0.952487\n",
      "[300]\ttrain-auc:0.960999\tvalid-auc:0.959622\n",
      "[400]\ttrain-auc:0.965869\tvalid-auc:0.964654\n",
      "[500]\ttrain-auc:0.969096\tvalid-auc:0.967938\n",
      "[600]\ttrain-auc:0.971429\tvalid-auc:0.970194\n",
      "[700]\ttrain-auc:0.973029\tvalid-auc:0.971615\n",
      "[800]\ttrain-auc:0.974225\tvalid-auc:0.972594\n",
      "[900]\ttrain-auc:0.975142\tvalid-auc:0.973307\n",
      "[1000]\ttrain-auc:0.976025\tvalid-auc:0.97393\n",
      "[1100]\ttrain-auc:0.976738\tvalid-auc:0.974431\n",
      "[1200]\ttrain-auc:0.977329\tvalid-auc:0.974827\n",
      "[1300]\ttrain-auc:0.977874\tvalid-auc:0.975098\n",
      "[1400]\ttrain-auc:0.978338\tvalid-auc:0.975397\n",
      "[1500]\ttrain-auc:0.978803\tvalid-auc:0.975702\n",
      "[1600]\ttrain-auc:0.979229\tvalid-auc:0.975891\n",
      "[1700]\ttrain-auc:0.979608\tvalid-auc:0.976122\n",
      "[1800]\ttrain-auc:0.979964\tvalid-auc:0.976316\n",
      "[1900]\ttrain-auc:0.980319\tvalid-auc:0.97651\n",
      "[2000]\ttrain-auc:0.980633\tvalid-auc:0.976666\n",
      "[2100]\ttrain-auc:0.98093\tvalid-auc:0.976769\n",
      "[2200]\ttrain-auc:0.981194\tvalid-auc:0.976907\n",
      "[2300]\ttrain-auc:0.98148\tvalid-auc:0.977034\n",
      "[2400]\ttrain-auc:0.981741\tvalid-auc:0.977175\n",
      "[2500]\ttrain-auc:0.982023\tvalid-auc:0.977269\n",
      "[2600]\ttrain-auc:0.982265\tvalid-auc:0.977363\n",
      "[2700]\ttrain-auc:0.982518\tvalid-auc:0.977504\n",
      "[2800]\ttrain-auc:0.982734\tvalid-auc:0.977606\n",
      "[2900]\ttrain-auc:0.982956\tvalid-auc:0.977727\n",
      "[3000]\ttrain-auc:0.983174\tvalid-auc:0.977806\n",
      "[3100]\ttrain-auc:0.983376\tvalid-auc:0.977903\n",
      "[3200]\ttrain-auc:0.983571\tvalid-auc:0.977986\n",
      "[3300]\ttrain-auc:0.98376\tvalid-auc:0.978055\n",
      "[3400]\ttrain-auc:0.983943\tvalid-auc:0.97815\n",
      "[3500]\ttrain-auc:0.984133\tvalid-auc:0.978256\n",
      "[3600]\ttrain-auc:0.984288\tvalid-auc:0.978318\n",
      "[3700]\ttrain-auc:0.984473\tvalid-auc:0.978343\n",
      "[3800]\ttrain-auc:0.984642\tvalid-auc:0.978415\n",
      "[3900]\ttrain-auc:0.984819\tvalid-auc:0.978481\n",
      "[4000]\ttrain-auc:0.984986\tvalid-auc:0.978583\n",
      "[4100]\ttrain-auc:0.98516\tvalid-auc:0.978642\n",
      "[4200]\ttrain-auc:0.985323\tvalid-auc:0.978711\n",
      "[4300]\ttrain-auc:0.985473\tvalid-auc:0.978766\n",
      "[4400]\ttrain-auc:0.985613\tvalid-auc:0.978848\n",
      "[4500]\ttrain-auc:0.985764\tvalid-auc:0.978905\n",
      "[4600]\ttrain-auc:0.985912\tvalid-auc:0.978981\n",
      "[4700]\ttrain-auc:0.98605\tvalid-auc:0.979006\n",
      "[4800]\ttrain-auc:0.986197\tvalid-auc:0.979022\n",
      "Stopping. Best iteration:\n",
      "[4771]\ttrain-auc:0.986161\tvalid-auc:0.979036\n",
      "\n",
      "('No ', 3)\n",
      "[0]\ttrain-auc:0.86756\tvalid-auc:0.86854\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 100 rounds.\n",
      "[100]\ttrain-auc:0.947676\tvalid-auc:0.946474\n",
      "[200]\ttrain-auc:0.954238\tvalid-auc:0.952823\n",
      "[300]\ttrain-auc:0.960975\tvalid-auc:0.960004\n",
      "[400]\ttrain-auc:0.965935\tvalid-auc:0.964907\n",
      "[500]\ttrain-auc:0.969157\tvalid-auc:0.967849\n",
      "[600]\ttrain-auc:0.971555\tvalid-auc:0.969948\n",
      "[700]\ttrain-auc:0.973185\tvalid-auc:0.971235\n",
      "[800]\ttrain-auc:0.97438\tvalid-auc:0.972119\n",
      "[900]\ttrain-auc:0.975369\tvalid-auc:0.972829\n",
      "[1000]\ttrain-auc:0.976144\tvalid-auc:0.973395\n",
      "[1100]\ttrain-auc:0.976849\tvalid-auc:0.973831\n",
      "[1200]\ttrain-auc:0.977472\tvalid-auc:0.9742\n",
      "[1300]\ttrain-auc:0.977987\tvalid-auc:0.974564\n",
      "[1400]\ttrain-auc:0.978427\tvalid-auc:0.97482\n",
      "[1500]\ttrain-auc:0.978912\tvalid-auc:0.97515\n",
      "[1600]\ttrain-auc:0.979329\tvalid-auc:0.975469\n",
      "[1700]\ttrain-auc:0.979678\tvalid-auc:0.975686\n",
      "[1800]\ttrain-auc:0.980033\tvalid-auc:0.975861\n",
      "[1900]\ttrain-auc:0.980408\tvalid-auc:0.976097\n",
      "[2000]\ttrain-auc:0.980737\tvalid-auc:0.976274\n",
      "[2100]\ttrain-auc:0.981037\tvalid-auc:0.976435\n",
      "[2200]\ttrain-auc:0.98132\tvalid-auc:0.976612\n",
      "[2300]\ttrain-auc:0.981574\tvalid-auc:0.976772\n",
      "[2400]\ttrain-auc:0.981842\tvalid-auc:0.976875\n",
      "[2500]\ttrain-auc:0.982101\tvalid-auc:0.976991\n",
      "[2600]\ttrain-auc:0.982321\tvalid-auc:0.977107\n",
      "[2700]\ttrain-auc:0.982553\tvalid-auc:0.977214\n",
      "[2800]\ttrain-auc:0.982761\tvalid-auc:0.977332\n",
      "[2900]\ttrain-auc:0.982977\tvalid-auc:0.977402\n",
      "[3000]\ttrain-auc:0.98316\tvalid-auc:0.977502\n",
      "[3100]\ttrain-auc:0.983399\tvalid-auc:0.977605\n",
      "[3200]\ttrain-auc:0.983575\tvalid-auc:0.977692\n",
      "[3300]\ttrain-auc:0.983774\tvalid-auc:0.97776\n",
      "[3400]\ttrain-auc:0.983963\tvalid-auc:0.977878\n",
      "[3500]\ttrain-auc:0.984146\tvalid-auc:0.977969\n",
      "[3600]\ttrain-auc:0.984327\tvalid-auc:0.97802\n",
      "[3700]\ttrain-auc:0.984517\tvalid-auc:0.978096\n",
      "[3800]\ttrain-auc:0.984703\tvalid-auc:0.978158\n",
      "[3900]\ttrain-auc:0.984856\tvalid-auc:0.978209\n",
      "[4000]\ttrain-auc:0.985022\tvalid-auc:0.978273\n",
      "[4100]\ttrain-auc:0.985165\tvalid-auc:0.978323\n",
      "[4200]\ttrain-auc:0.985333\tvalid-auc:0.978381\n",
      "[4300]\ttrain-auc:0.985477\tvalid-auc:0.978431\n",
      "[4400]\ttrain-auc:0.985645\tvalid-auc:0.978512\n",
      "Stopping. Best iteration:\n",
      "[4399]\ttrain-auc:0.985645\tvalid-auc:0.978515\n",
      "\n",
      "('No ', 4)\n",
      "[0]\ttrain-auc:0.857992\tvalid-auc:0.841273\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 100 rounds.\n",
      "[100]\ttrain-auc:0.950269\tvalid-auc:0.942954\n",
      "[200]\ttrain-auc:0.955768\tvalid-auc:0.948775\n",
      "[300]\ttrain-auc:0.960919\tvalid-auc:0.954334\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[400]\ttrain-auc:0.966566\tvalid-auc:0.960532\n",
      "[500]\ttrain-auc:0.969946\tvalid-auc:0.964252\n",
      "[600]\ttrain-auc:0.972178\tvalid-auc:0.966595\n",
      "[700]\ttrain-auc:0.973598\tvalid-auc:0.968056\n",
      "[800]\ttrain-auc:0.974681\tvalid-auc:0.969203\n",
      "[900]\ttrain-auc:0.975591\tvalid-auc:0.970121\n",
      "[1000]\ttrain-auc:0.976345\tvalid-auc:0.970785\n",
      "[1100]\ttrain-auc:0.977021\tvalid-auc:0.97147\n",
      "[1200]\ttrain-auc:0.977603\tvalid-auc:0.971981\n",
      "[1300]\ttrain-auc:0.978139\tvalid-auc:0.972401\n",
      "[1400]\ttrain-auc:0.978637\tvalid-auc:0.972807\n",
      "[1500]\ttrain-auc:0.979089\tvalid-auc:0.973191\n",
      "[1600]\ttrain-auc:0.979501\tvalid-auc:0.973444\n",
      "[1700]\ttrain-auc:0.979913\tvalid-auc:0.973731\n",
      "[1800]\ttrain-auc:0.980288\tvalid-auc:0.973937\n",
      "[1900]\ttrain-auc:0.980654\tvalid-auc:0.974175\n",
      "[2000]\ttrain-auc:0.980977\tvalid-auc:0.974299\n",
      "[2100]\ttrain-auc:0.981267\tvalid-auc:0.974383\n",
      "[2200]\ttrain-auc:0.98157\tvalid-auc:0.974537\n",
      "[2300]\ttrain-auc:0.981845\tvalid-auc:0.974714\n",
      "[2400]\ttrain-auc:0.982111\tvalid-auc:0.974811\n",
      "[2500]\ttrain-auc:0.982372\tvalid-auc:0.974878\n",
      "[2600]\ttrain-auc:0.982622\tvalid-auc:0.975005\n",
      "[2700]\ttrain-auc:0.982856\tvalid-auc:0.975128\n",
      "[2800]\ttrain-auc:0.983091\tvalid-auc:0.975188\n",
      "[2900]\ttrain-auc:0.983324\tvalid-auc:0.975331\n",
      "[3000]\ttrain-auc:0.983531\tvalid-auc:0.975391\n",
      "[3100]\ttrain-auc:0.983734\tvalid-auc:0.975497\n",
      "[3200]\ttrain-auc:0.98394\tvalid-auc:0.975597\n",
      "[3300]\ttrain-auc:0.984147\tvalid-auc:0.975684\n",
      "[3400]\ttrain-auc:0.98434\tvalid-auc:0.975716\n",
      "[3500]\ttrain-auc:0.984512\tvalid-auc:0.975787\n",
      "[3600]\ttrain-auc:0.984686\tvalid-auc:0.975829\n",
      "[3700]\ttrain-auc:0.984874\tvalid-auc:0.975909\n",
      "[3800]\ttrain-auc:0.985049\tvalid-auc:0.975946\n",
      "[3900]\ttrain-auc:0.985196\tvalid-auc:0.975988\n",
      "[4000]\ttrain-auc:0.985336\tvalid-auc:0.976032\n",
      "[4100]\ttrain-auc:0.985485\tvalid-auc:0.976082\n",
      "[4200]\ttrain-auc:0.985626\tvalid-auc:0.976125\n",
      "[4300]\ttrain-auc:0.985775\tvalid-auc:0.976172\n",
      "[4400]\ttrain-auc:0.985927\tvalid-auc:0.976226\n",
      "[4500]\ttrain-auc:0.986072\tvalid-auc:0.976297\n",
      "[4600]\ttrain-auc:0.986222\tvalid-auc:0.976339\n",
      "[4700]\ttrain-auc:0.986358\tvalid-auc:0.976363\n",
      "[4800]\ttrain-auc:0.986484\tvalid-auc:0.97639\n",
      "[4900]\ttrain-auc:0.986624\tvalid-auc:0.976421\n",
      "[5000]\ttrain-auc:0.986738\tvalid-auc:0.976458\n",
      "[5100]\ttrain-auc:0.986854\tvalid-auc:0.976493\n",
      "[5200]\ttrain-auc:0.986973\tvalid-auc:0.976536\n",
      "[5300]\ttrain-auc:0.987098\tvalid-auc:0.976554\n",
      "[5400]\ttrain-auc:0.987209\tvalid-auc:0.976575\n",
      "[5500]\ttrain-auc:0.987319\tvalid-auc:0.976595\n",
      "Stopping. Best iteration:\n",
      "[5472]\ttrain-auc:0.987284\tvalid-auc:0.976605\n",
      "\n",
      "('No ', 5)\n",
      "[0]\ttrain-auc:0.852062\tvalid-auc:0.866243\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 100 rounds.\n",
      "[100]\ttrain-auc:0.948599\tvalid-auc:0.952029\n",
      "[200]\ttrain-auc:0.954194\tvalid-auc:0.957366\n",
      "[300]\ttrain-auc:0.959708\tvalid-auc:0.961981\n",
      "[400]\ttrain-auc:0.965721\tvalid-auc:0.966602\n",
      "[500]\ttrain-auc:0.969234\tvalid-auc:0.969218\n",
      "[600]\ttrain-auc:0.971517\tvalid-auc:0.971027\n",
      "[700]\ttrain-auc:0.972912\tvalid-auc:0.972078\n",
      "[800]\ttrain-auc:0.974052\tvalid-auc:0.972885\n",
      "[900]\ttrain-auc:0.974965\tvalid-auc:0.973484\n",
      "[1000]\ttrain-auc:0.975738\tvalid-auc:0.973942\n",
      "[1100]\ttrain-auc:0.976445\tvalid-auc:0.974368\n",
      "[1200]\ttrain-auc:0.97703\tvalid-auc:0.974664\n",
      "[1300]\ttrain-auc:0.977553\tvalid-auc:0.974885\n",
      "[1400]\ttrain-auc:0.978008\tvalid-auc:0.975148\n",
      "[1500]\ttrain-auc:0.97846\tvalid-auc:0.975439\n",
      "[1600]\ttrain-auc:0.978867\tvalid-auc:0.975648\n",
      "[1700]\ttrain-auc:0.979274\tvalid-auc:0.975903\n",
      "[1800]\ttrain-auc:0.979651\tvalid-auc:0.976089\n",
      "[1900]\ttrain-auc:0.979989\tvalid-auc:0.976227\n",
      "[2000]\ttrain-auc:0.980334\tvalid-auc:0.976348\n",
      "[2100]\ttrain-auc:0.980667\tvalid-auc:0.976491\n",
      "[2200]\ttrain-auc:0.980973\tvalid-auc:0.976619\n",
      "[2300]\ttrain-auc:0.981237\tvalid-auc:0.976729\n",
      "[2400]\ttrain-auc:0.981493\tvalid-auc:0.976807\n",
      "[2500]\ttrain-auc:0.981751\tvalid-auc:0.976891\n",
      "[2600]\ttrain-auc:0.981976\tvalid-auc:0.976982\n",
      "[2700]\ttrain-auc:0.982217\tvalid-auc:0.977072\n",
      "[2800]\ttrain-auc:0.982459\tvalid-auc:0.977178\n",
      "[2900]\ttrain-auc:0.982695\tvalid-auc:0.977262\n",
      "[3000]\ttrain-auc:0.982901\tvalid-auc:0.977338\n",
      "[3100]\ttrain-auc:0.983119\tvalid-auc:0.977399\n",
      "[3200]\ttrain-auc:0.983302\tvalid-auc:0.977467\n",
      "[3300]\ttrain-auc:0.983496\tvalid-auc:0.977524\n",
      "[3400]\ttrain-auc:0.98371\tvalid-auc:0.977613\n",
      "[3500]\ttrain-auc:0.983889\tvalid-auc:0.977678\n",
      "[3600]\ttrain-auc:0.984073\tvalid-auc:0.977717\n",
      "[3700]\ttrain-auc:0.984246\tvalid-auc:0.977773\n",
      "[3800]\ttrain-auc:0.984408\tvalid-auc:0.977815\n",
      "[3900]\ttrain-auc:0.984571\tvalid-auc:0.977866\n",
      "[4000]\ttrain-auc:0.984736\tvalid-auc:0.977931\n",
      "[4100]\ttrain-auc:0.984899\tvalid-auc:0.977996\n",
      "[4200]\ttrain-auc:0.985071\tvalid-auc:0.97806\n",
      "[4300]\ttrain-auc:0.98521\tvalid-auc:0.978107\n",
      "[4400]\ttrain-auc:0.985371\tvalid-auc:0.978178\n",
      "[4500]\ttrain-auc:0.985539\tvalid-auc:0.978232\n",
      "[4600]\ttrain-auc:0.985672\tvalid-auc:0.978273\n",
      "[4700]\ttrain-auc:0.985806\tvalid-auc:0.978305\n",
      "[4800]\ttrain-auc:0.985932\tvalid-auc:0.978375\n",
      "[4900]\ttrain-auc:0.986071\tvalid-auc:0.978411\n",
      "[5000]\ttrain-auc:0.986208\tvalid-auc:0.97843\n",
      "[5100]\ttrain-auc:0.986331\tvalid-auc:0.978486\n",
      "[5200]\ttrain-auc:0.986461\tvalid-auc:0.978512\n",
      "[5300]\ttrain-auc:0.98659\tvalid-auc:0.978559\n",
      "[5400]\ttrain-auc:0.986711\tvalid-auc:0.978586\n",
      "[5500]\ttrain-auc:0.986819\tvalid-auc:0.978613\n",
      "[5600]\ttrain-auc:0.986932\tvalid-auc:0.978639\n",
      "[5700]\ttrain-auc:0.98706\tvalid-auc:0.978679\n",
      "[5800]\ttrain-auc:0.987174\tvalid-auc:0.978718\n",
      "[5900]\ttrain-auc:0.987289\tvalid-auc:0.978752\n",
      "[6000]\ttrain-auc:0.987396\tvalid-auc:0.978751\n",
      "[6100]\ttrain-auc:0.98749\tvalid-auc:0.978773\n",
      "[6200]\ttrain-auc:0.987589\tvalid-auc:0.978792\n",
      "[6300]\ttrain-auc:0.987706\tvalid-auc:0.978821\n",
      "[6400]\ttrain-auc:0.987798\tvalid-auc:0.97887\n",
      "[6500]\ttrain-auc:0.987923\tvalid-auc:0.978892\n",
      "[6600]\ttrain-auc:0.988029\tvalid-auc:0.978912\n",
      "[6700]\ttrain-auc:0.98812\tvalid-auc:0.978931\n",
      "[6800]\ttrain-auc:0.988217\tvalid-auc:0.978955\n",
      "[6900]\ttrain-auc:0.988326\tvalid-auc:0.978967\n",
      "[7000]\ttrain-auc:0.988421\tvalid-auc:0.979007\n",
      "[7100]\ttrain-auc:0.98852\tvalid-auc:0.979044\n",
      "[7200]\ttrain-auc:0.988607\tvalid-auc:0.979048\n",
      "Stopping. Best iteration:\n",
      "[7119]\ttrain-auc:0.988541\tvalid-auc:0.979052\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"K-fold xgb training!\")\n",
    "kf1 = KFold(n_splits=5, shuffle=True, random_state=998)\n",
    "test['score'] = 0\n",
    "i = 0\n",
    "bst = 0\n",
    "for trn, tst in kf1.split(X):\n",
    "    i += 1\n",
    "    print(\"No \",i)\n",
    "    trn_X, tst_X = X[trn], X[tst]\n",
    "    trn_Y, tst_Y = y[trn], y[tst]\n",
    "    d_train = xgb.DMatrix(trn_X, trn_Y) \n",
    "    d_valid = xgb.DMatrix(tst_X, tst_Y) \n",
    "    watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n",
    "    xgb_model = xgb.train(xgbparams, d_train, 100000, watchlist, early_stopping_rounds=100,maximize=True, verbose_eval=100)\n",
    "    test['score']  += xgb_model.predict(xgb.DMatrix(test[pred].values), ntree_limit= xgb_model.best_ntree_limit+150) / (2 * kfold)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbparams = {'eta': 0.2, \n",
    "          'tree_method': \"auto\", \n",
    "          'max_depth': 8, \n",
    "          'subsample': 0.9, \n",
    "          'colsample_bytree': 0.1,  \n",
    "          'min_child_weight':1,\n",
    "          'objective': 'binary:logistic', \n",
    "          'eval_metric': 'auc', \n",
    "          'random_state': 99,\n",
    "          'silent': True}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K-fold xgb training!\n",
      "('No ', 1)\n",
      "[0]\ttrain-auc:0.904146\tvalid-auc:0.911153\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 100 rounds.\n",
      "[100]\ttrain-auc:0.99394\tvalid-auc:0.978098\n",
      "[200]\ttrain-auc:0.997267\tvalid-auc:0.978685\n",
      "Stopping. Best iteration:\n",
      "[180]\ttrain-auc:0.996938\tvalid-auc:0.979004\n",
      "\n",
      "('No ', 2)\n",
      "[0]\ttrain-auc:0.905312\tvalid-auc:0.905185\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 100 rounds.\n",
      "[100]\ttrain-auc:0.993878\tvalid-auc:0.977893\n",
      "[200]\ttrain-auc:0.997329\tvalid-auc:0.979318\n",
      "Stopping. Best iteration:\n",
      "[185]\ttrain-auc:0.997134\tvalid-auc:0.979615\n",
      "\n",
      "('No ', 3)\n",
      "[0]\ttrain-auc:0.905311\tvalid-auc:0.901288\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 100 rounds.\n",
      "[100]\ttrain-auc:0.994159\tvalid-auc:0.977746\n",
      "[200]\ttrain-auc:0.997151\tvalid-auc:0.978909\n",
      "Stopping. Best iteration:\n",
      "[183]\ttrain-auc:0.996894\tvalid-auc:0.979219\n",
      "\n",
      "('No ', 4)\n",
      "[0]\ttrain-auc:0.900148\tvalid-auc:0.884491\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 100 rounds.\n",
      "[100]\ttrain-auc:0.994303\tvalid-auc:0.975121\n",
      "[200]\ttrain-auc:0.997294\tvalid-auc:0.975712\n",
      "[300]\ttrain-auc:0.998366\tvalid-auc:0.975512\n",
      "Stopping. Best iteration:\n",
      "[218]\ttrain-auc:0.997618\tvalid-auc:0.9761\n",
      "\n",
      "('No ', 5)\n",
      "[0]\ttrain-auc:0.89517\tvalid-auc:0.903726\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 100 rounds.\n",
      "[100]\ttrain-auc:0.994102\tvalid-auc:0.979111\n",
      "[200]\ttrain-auc:0.997358\tvalid-auc:0.97887\n",
      "Stopping. Best iteration:\n",
      "[140]\ttrain-auc:0.996146\tvalid-auc:0.979381\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"K-fold xgb training!\")\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=998)\n",
    "i = 0\n",
    "for trn, tst in kf.split(X):\n",
    "    i += 1\n",
    "    print(\"No \",i)\n",
    "    trn_X, tst_X = X[trn], X[tst]\n",
    "    trn_Y, tst_Y = y[trn], y[tst]\n",
    "    d_train = xgb.DMatrix(trn_X, trn_Y) \n",
    "    d_valid = xgb.DMatrix(tst_X, tst_Y) \n",
    "    watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n",
    "    xgb_model = xgb.train(xgbparams, d_train, 100000, watchlist, early_stopping_rounds=100,maximize=True, verbose_eval=100)\n",
    "    test['score']  += xgb_model.predict(xgb.DMatrix(test[pred].values), ntree_limit= xgb_model.best_ntree_limit+150) / (2 * kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'test' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-c30594a31eed>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtest\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'id'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'score'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'sub_xgb_final.csv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'test' is not defined"
     ]
    }
   ],
   "source": [
    "test[['id', 'score']].to_csv('sub_xgb_final.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = [1,2,3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = [1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}