SparkML - 02 Credit Default.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Launch spark session behind the jupyter notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls -l $SPARK_HOME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spark version:  2.2.0\n"
     ]
    }
   ],
   "source": [
    "# Note: set SPARK_HOME to Spark binaries before launching the Jupyter session.\n",
    "import os, sys\n",
    "SPARK_HOME = os.environ['SPARK_HOME']\n",
    "sys.path.insert(0, os.path.join(SPARK_HOME, \"python\", \"lib\", \"py4j-0.10.4-src.zip\"))\n",
    "sys.path.insert(0, os.path.join(SPARK_HOME, \"python\"))\n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.getOrCreate()\n",
    "print(\"Spark version: \", spark.version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'http://192.168.1.6:4040'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sparkContext.uiWebUrl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libararies "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import StringIndexer, OneHotEncoder, VectorAssembler\n",
    "from pyspark.ml.pipeline import Pipeline\n",
    "\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml import evaluation\n",
    "from pyspark.sql.functions import * \n",
    "\n",
    "import pandas as pd\n",
    "import pyspark\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('0.21.0', '1.13.3', '2.2.0')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.__version__, np.__version__,pyspark.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check version of the libraries. For this notebook, I am using Spark 2.2.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Dataset\n",
    "\n",
    "You can download the dataset from [here](https://github.com/abulbasar/data/blob/master/credit-default.csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of records:  1000\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>checking_balance</th>\n",
       "      <th>months_loan_duration</th>\n",
       "      <th>credit_history</th>\n",
       "      <th>purpose</th>\n",
       "      <th>amount</th>\n",
       "      <th>savings_balance</th>\n",
       "      <th>employment_length</th>\n",
       "      <th>installment_rate</th>\n",
       "      <th>personal_status</th>\n",
       "      <th>other_debtors</th>\n",
       "      <th>...</th>\n",
       "      <th>property</th>\n",
       "      <th>age</th>\n",
       "      <th>installment_plan</th>\n",
       "      <th>housing</th>\n",
       "      <th>existing_credits</th>\n",
       "      <th>default</th>\n",
       "      <th>dependents</th>\n",
       "      <th>telephone</th>\n",
       "      <th>foreign_worker</th>\n",
       "      <th>job</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt; 0 DM</td>\n",
       "      <td>6</td>\n",
       "      <td>critical</td>\n",
       "      <td>radio/tv</td>\n",
       "      <td>1169</td>\n",
       "      <td>unknown</td>\n",
       "      <td>&gt; 7 yrs</td>\n",
       "      <td>4</td>\n",
       "      <td>single male</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>real estate</td>\n",
       "      <td>67</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1 - 200 DM</td>\n",
       "      <td>48</td>\n",
       "      <td>repaid</td>\n",
       "      <td>radio/tv</td>\n",
       "      <td>5951</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>1 - 4 yrs</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>real estate</td>\n",
       "      <td>22</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unknown</td>\n",
       "      <td>12</td>\n",
       "      <td>critical</td>\n",
       "      <td>education</td>\n",
       "      <td>2096</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>4 - 7 yrs</td>\n",
       "      <td>2</td>\n",
       "      <td>single male</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>real estate</td>\n",
       "      <td>49</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>unskilled resident</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt; 0 DM</td>\n",
       "      <td>42</td>\n",
       "      <td>repaid</td>\n",
       "      <td>furniture</td>\n",
       "      <td>7882</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>4 - 7 yrs</td>\n",
       "      <td>2</td>\n",
       "      <td>single male</td>\n",
       "      <td>guarantor</td>\n",
       "      <td>...</td>\n",
       "      <td>building society savings</td>\n",
       "      <td>45</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt; 0 DM</td>\n",
       "      <td>24</td>\n",
       "      <td>delayed</td>\n",
       "      <td>car (new)</td>\n",
       "      <td>4870</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>1 - 4 yrs</td>\n",
       "      <td>3</td>\n",
       "      <td>single male</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>unknown/none</td>\n",
       "      <td>53</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  checking_balance  months_loan_duration credit_history    purpose  amount  \\\n",
       "0           < 0 DM                     6       critical   radio/tv    1169   \n",
       "1       1 - 200 DM                    48         repaid   radio/tv    5951   \n",
       "2          unknown                    12       critical  education    2096   \n",
       "3           < 0 DM                    42         repaid  furniture    7882   \n",
       "4           < 0 DM                    24        delayed  car (new)    4870   \n",
       "\n",
       "  savings_balance employment_length  installment_rate personal_status  \\\n",
       "0         unknown           > 7 yrs                 4     single male   \n",
       "1        < 100 DM         1 - 4 yrs                 2          female   \n",
       "2        < 100 DM         4 - 7 yrs                 2     single male   \n",
       "3        < 100 DM         4 - 7 yrs                 2     single male   \n",
       "4        < 100 DM         1 - 4 yrs                 3     single male   \n",
       "\n",
       "  other_debtors         ...                          property age  \\\n",
       "0          none         ...                       real estate  67   \n",
       "1          none         ...                       real estate  22   \n",
       "2          none         ...                       real estate  49   \n",
       "3     guarantor         ...          building society savings  45   \n",
       "4          none         ...                      unknown/none  53   \n",
       "\n",
       "   installment_plan   housing existing_credits  default  dependents  \\\n",
       "0              none       own                2        1           1   \n",
       "1              none       own                1        2           1   \n",
       "2              none       own                1        1           2   \n",
       "3              none  for free                1        1           2   \n",
       "4              none  for free                2        2           2   \n",
       "\n",
       "   telephone foreign_worker                 job  \n",
       "0        yes            yes    skilled employee  \n",
       "1       none            yes    skilled employee  \n",
       "2       none            yes  unskilled resident  \n",
       "3       none            yes    skilled employee  \n",
       "4       none            yes    skilled employee  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "credit = spark.read.options(header = True, inferSchema = True).csv(\"/data/credit-default.csv\").cache()\n",
    "print(\"Total number of records: \", credit.count())\n",
    "credit.limit(10).toPandas().head() \n",
    "# Taking 10 samples records from spark dtaframe into a Pandas dataframe to display the values\n",
    "# I prefer the pandas dataframe display to that by spark dataframe show function."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "View the schema "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- checking_balance: string (nullable = true)\n",
      " |-- months_loan_duration: integer (nullable = true)\n",
      " |-- credit_history: string (nullable = true)\n",
      " |-- purpose: string (nullable = true)\n",
      " |-- amount: integer (nullable = true)\n",
      " |-- savings_balance: string (nullable = true)\n",
      " |-- employment_length: string (nullable = true)\n",
      " |-- installment_rate: integer (nullable = true)\n",
      " |-- personal_status: string (nullable = true)\n",
      " |-- other_debtors: string (nullable = true)\n",
      " |-- residence_history: integer (nullable = true)\n",
      " |-- property: string (nullable = true)\n",
      " |-- age: integer (nullable = true)\n",
      " |-- installment_plan: string (nullable = true)\n",
      " |-- housing: string (nullable = true)\n",
      " |-- existing_credits: integer (nullable = true)\n",
      " |-- default: integer (nullable = true)\n",
      " |-- dependents: integer (nullable = true)\n",
      " |-- telephone: string (nullable = true)\n",
      " |-- foreign_worker: string (nullable = true)\n",
      " |-- job: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "credit.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "As I can see, there are number of columns of string type - checking_balance, credit_history etc.\n",
    "\n",
    "Let me define a function that take a catgorical column and pass it through StringIndexer and OneHotEncoder it gives back a dataframe with same column name as the original categorical column. It reurns a new dataframe that contains categorical column replaced by OneHotEncoded vector. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find all columns of String datatype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Transform each string column type into OneHotEncoded value and collect distinct values for each categorical column in list as shown below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['checking_balance',\n",
       " 'months_loan_duration',\n",
       " 'credit_history',\n",
       " 'purpose',\n",
       " 'amount',\n",
       " 'savings_balance',\n",
       " 'employment_length',\n",
       " 'installment_rate',\n",
       " 'personal_status',\n",
       " 'other_debtors',\n",
       " 'residence_history',\n",
       " 'property',\n",
       " 'age',\n",
       " 'installment_plan',\n",
       " 'housing',\n",
       " 'existing_credits',\n",
       " 'dependents',\n",
       " 'telephone',\n",
       " 'foreign_worker',\n",
       " 'job']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = credit.columns\n",
    "cols.remove(\"default\")\n",
    "cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml import Model, Estimator \n",
    "\n",
    "class DFOneHotEncoderModel(Model):\n",
    "    \n",
    "    def get_col_labels(self):\n",
    "        \n",
    "        cols = []\n",
    "        feature_columns = [c for c in self.columns if not c == self.label_column]\n",
    "        \n",
    "        for col in feature_columns:\n",
    "            if col in self.categorical_fields:\n",
    "                string_indexer, _ = self.categorical_fields[col]\n",
    "                values = string_indexer.labels\n",
    "                values = values[:-1] if self.drop_last else values\n",
    "                values = [col + \"_\" + v for v in values]\n",
    "                cols.extend(values)\n",
    "            else:\n",
    "                cols.append(col) \n",
    "            \n",
    "        return cols\n",
    "    \n",
    "    def transform(self, df, params= None):\n",
    "        \n",
    "        for colname in self.categorical_fields:\n",
    "            string_indexer, one_hot_encoder = self.categorical_fields[colname]\n",
    "        \n",
    "            df = string_indexer.transform(df)\n",
    "            df = df.drop(colname)\n",
    "            df = df.withColumnRenamed(colname + \"_idx\", colname)\n",
    "\n",
    "            if one_hot_encoder:\n",
    "                df = one_hot_encoder.transform(df)\n",
    "                df = df.drop(colname)\n",
    "                df = df.withColumnRenamed(colname + \"_ohe\", colname)\n",
    "                \n",
    "        return df\n",
    "        \n",
    "class DFOneHotEncoder(Estimator):\n",
    "    \n",
    "    def __init__(self, label_column, categorical_fields= None,  one_hot = True, drop_last = True):\n",
    "        self.categorical_fields = None\n",
    "        self.one_hot = one_hot\n",
    "        self.drop_last = drop_last\n",
    "        self.label_column = label_column \n",
    "        \n",
    "        if not categorical_fields is None:\n",
    "            self.categorical_fields = dict([(c, None) for c in categorical_fields])     \n",
    "\n",
    "    def fit(self, df):\n",
    "        cols = df.columns\n",
    "        if self.categorical_fields is None:\n",
    "            self.categorical_fields = dict([(col, None) for col, dtype in df.dtypes if dtype == \"string\"])\n",
    "        \n",
    "        \n",
    "        for colname in self.categorical_fields:\n",
    "            string_indexer = StringIndexer(inputCol=colname, outputCol= colname + \"_idx\").fit(df)\n",
    "            \n",
    "            one_hot_encoder = None\n",
    "            if self.one_hot:\n",
    "                one_hot_encoder = OneHotEncoder(inputCol=colname\n",
    "                                            , outputCol=colname + \"_ohe\" , dropLast = self.drop_last)\n",
    "\n",
    "            self.categorical_fields[colname] = (string_indexer, one_hot_encoder)\n",
    "            \n",
    "\n",
    "        model = DFOneHotEncoderModel()\n",
    "        model.categorical_fields = self.categorical_fields\n",
    "        model.one_hot = self.one_hot\n",
    "        model.drop_last = self.drop_last\n",
    "        model.columns = cols\n",
    "        model.label_column = self.label_column\n",
    "        \n",
    "        return model\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('months_loan_duration', 'int'), ('amount', 'int'), ('installment_rate', 'int'), ('residence_history', 'int'), ('age', 'int'), ('existing_credits', 'int'), ('default', 'int'), ('dependents', 'int'), ('checking_balance', 'vector'), ('credit_history', 'vector'), ('purpose', 'vector'), ('savings_balance', 'vector'), ('employment_length', 'vector'), ('personal_status', 'vector'), ('other_debtors', 'vector'), ('property', 'vector'), ('installment_plan', 'vector'), ('housing', 'vector'), ('telephone', 'vector'), ('foreign_worker', 'vector'), ('job', 'vector')]\n",
      "\n",
      "\n",
      "['checking_balance_unknown', 'checking_balance_< 0 DM', 'checking_balance_1 - 200 DM', 'months_loan_duration', 'credit_history_repaid', 'credit_history_critical', 'credit_history_delayed', 'credit_history_fully repaid this bank', 'purpose_radio/tv', 'purpose_car (new)', 'purpose_furniture', 'purpose_car (used)', 'purpose_business', 'purpose_education', 'purpose_repairs', 'purpose_others', 'purpose_domestic appliances', 'amount', 'savings_balance_< 100 DM', 'savings_balance_unknown', 'savings_balance_101 - 500 DM', 'savings_balance_501 - 1000 DM', 'employment_length_1 - 4 yrs', 'employment_length_> 7 yrs', 'employment_length_4 - 7 yrs', 'employment_length_0 - 1 yrs', 'installment_rate', 'personal_status_single male', 'personal_status_female', 'personal_status_married male', 'other_debtors_none', 'other_debtors_guarantor', 'residence_history', 'property_other', 'property_real estate', 'property_building society savings', 'age', 'installment_plan_none', 'installment_plan_bank', 'housing_own', 'housing_rent', 'existing_credits', 'dependents', 'telephone_none', 'foreign_worker_yes', 'job_skilled employee', 'job_unskilled resident', 'job_mangement self-employed']\n"
     ]
    }
   ],
   "source": [
    "model = DFOneHotEncoder(label_column = \"default\").fit(credit)\n",
    "df = model.transform(credit)\n",
    "print(df.dtypes)\n",
    "print(\"\\n\")\n",
    "print(model.get_col_labels())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Verify that all columns in df is either of numeric or numeric vector type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- months_loan_duration: integer (nullable = true)\n",
      " |-- amount: integer (nullable = true)\n",
      " |-- installment_rate: integer (nullable = true)\n",
      " |-- residence_history: integer (nullable = true)\n",
      " |-- age: integer (nullable = true)\n",
      " |-- existing_credits: integer (nullable = true)\n",
      " |-- default: integer (nullable = true)\n",
      " |-- dependents: integer (nullable = true)\n",
      " |-- checking_balance: vector (nullable = true)\n",
      " |-- credit_history: vector (nullable = true)\n",
      " |-- purpose: vector (nullable = true)\n",
      " |-- savings_balance: vector (nullable = true)\n",
      " |-- employment_length: vector (nullable = true)\n",
      " |-- personal_status: vector (nullable = true)\n",
      " |-- other_debtors: vector (nullable = true)\n",
      " |-- property: vector (nullable = true)\n",
      " |-- installment_plan: vector (nullable = true)\n",
      " |-- housing: vector (nullable = true)\n",
      " |-- telephone: vector (nullable = true)\n",
      " |-- foreign_worker: vector (nullable = true)\n",
      " |-- job: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a list of columns except the label column"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use a vector assembler to transform all features into a single feature column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>features</th>\n",
       "      <th>default</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(0.0, 1.0, 0.0, 6.0, 0.0, 1.0, 0.0, 0.0, 1.0, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(0.0, 0.0, 1.0, 48.0, 1.0, 0.0, 0.0, 0.0, 1.0,...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(1.0, 0.0, 0.0, 12.0, 0.0, 1.0, 0.0, 0.0, 0.0,...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(0.0, 1.0, 0.0, 42.0, 1.0, 0.0, 0.0, 0.0, 0.0,...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(0.0, 1.0, 0.0, 24.0, 0.0, 0.0, 1.0, 0.0, 0.0,...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            features  default\n",
       "0  (0.0, 1.0, 0.0, 6.0, 0.0, 1.0, 0.0, 0.0, 1.0, ...        1\n",
       "1  (0.0, 0.0, 1.0, 48.0, 1.0, 0.0, 0.0, 0.0, 1.0,...        2\n",
       "2  (1.0, 0.0, 0.0, 12.0, 0.0, 1.0, 0.0, 0.0, 0.0,...        1\n",
       "3  (0.0, 1.0, 0.0, 42.0, 1.0, 0.0, 0.0, 0.0, 0.0,...        1\n",
       "4  (0.0, 1.0, 0.0, 24.0, 0.0, 0.0, 1.0, 0.0, 0.0,...        2"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_vect = VectorAssembler(inputCols = cols, outputCol=\"features\").transform(df)\n",
    "df_vect.select(\"features\", \"default\").limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let me spot check whether OneHotEncode worked ok."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(checking_balance='< 0 DM', months_loan_duration=6, credit_history='critical', purpose='radio/tv', amount=1169, savings_balance='unknown', employment_length='> 7 yrs', installment_rate=4, personal_status='single male', other_debtors='none', residence_history=4, property='real estate', age=67, installment_plan='none', housing='own', existing_credits=2, default=1, dependents=1, telephone='yes', foreign_worker='yes', job='skilled employee')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "credit.first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>checking_balance_unknown</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>checking_balance_&lt; 0 DM</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>checking_balance_1 - 200 DM</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>months_loan_duration</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>credit_history_repaid</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>credit_history_critical</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>credit_history_delayed</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>credit_history_fully repaid this bank</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>purpose_radio/tv</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>purpose_car (new)</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>purpose_furniture</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>purpose_car (used)</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>purpose_business</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>purpose_education</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>purpose_repairs</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>purpose_others</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>purpose_domestic appliances</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>amount</td>\n",
       "      <td>1169.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>savings_balance_&lt; 100 DM</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>savings_balance_unknown</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>savings_balance_101 - 500 DM</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>savings_balance_501 - 1000 DM</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>employment_length_1 - 4 yrs</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>employment_length_&gt; 7 yrs</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>employment_length_4 - 7 yrs</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>employment_length_0 - 1 yrs</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>installment_rate</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>personal_status_single male</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>personal_status_female</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>personal_status_married male</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>other_debtors_none</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>other_debtors_guarantor</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>residence_history</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>property_other</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>property_real estate</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>property_building society savings</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>age</td>\n",
       "      <td>67.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>installment_plan_none</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>installment_plan_bank</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>housing_own</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>housing_rent</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>existing_credits</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>dependents</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>telephone_none</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>foreign_worker_yes</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>job_skilled employee</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>job_unskilled resident</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>job_mangement self-employed</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  feature   value\n",
       "0                checking_balance_unknown     0.0\n",
       "1                 checking_balance_< 0 DM     1.0\n",
       "2             checking_balance_1 - 200 DM     0.0\n",
       "3                    months_loan_duration     6.0\n",
       "4                   credit_history_repaid     0.0\n",
       "5                 credit_history_critical     1.0\n",
       "6                  credit_history_delayed     0.0\n",
       "7   credit_history_fully repaid this bank     0.0\n",
       "8                        purpose_radio/tv     1.0\n",
       "9                       purpose_car (new)     0.0\n",
       "10                      purpose_furniture     0.0\n",
       "11                     purpose_car (used)     0.0\n",
       "12                       purpose_business     0.0\n",
       "13                      purpose_education     0.0\n",
       "14                        purpose_repairs     0.0\n",
       "15                         purpose_others     0.0\n",
       "16            purpose_domestic appliances     0.0\n",
       "17                                 amount  1169.0\n",
       "18               savings_balance_< 100 DM     0.0\n",
       "19                savings_balance_unknown     1.0\n",
       "20           savings_balance_101 - 500 DM     0.0\n",
       "21          savings_balance_501 - 1000 DM     0.0\n",
       "22            employment_length_1 - 4 yrs     0.0\n",
       "23              employment_length_> 7 yrs     1.0\n",
       "24            employment_length_4 - 7 yrs     0.0\n",
       "25            employment_length_0 - 1 yrs     0.0\n",
       "26                       installment_rate     4.0\n",
       "27            personal_status_single male     1.0\n",
       "28                 personal_status_female     0.0\n",
       "29           personal_status_married male     0.0\n",
       "30                     other_debtors_none     1.0\n",
       "31                other_debtors_guarantor     0.0\n",
       "32                      residence_history     4.0\n",
       "33                         property_other     0.0\n",
       "34                   property_real estate     1.0\n",
       "35      property_building society savings     0.0\n",
       "36                                    age    67.0\n",
       "37                  installment_plan_none     1.0\n",
       "38                  installment_plan_bank     0.0\n",
       "39                            housing_own     1.0\n",
       "40                           housing_rent     0.0\n",
       "41                       existing_credits     2.0\n",
       "42                             dependents     1.0\n",
       "43                         telephone_none     0.0\n",
       "44                     foreign_worker_yes     1.0\n",
       "45                   job_skilled employee     1.0\n",
       "46                 job_unskilled resident     0.0\n",
       "47            job_mangement self-employed     0.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({\"feature\": model.get_col_labels(), \"value\": df_vect.select(\"features\").first().features})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(704, 296)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train, df_test = df_vect.randomSplit(weights=[0.7, 0.3], seed=1)\n",
    "df_train.count(), df_test.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build a RandomForest Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest = RandomForestClassifier(labelCol=\"default\", featuresCol=\"features\", seed = 123)\n",
    "forest_model = forest.fit(df_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run prediction on the whole dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+----------------+-----------------+---+----------------+-------+----------+----------------+--------------+-------------+---------------+-----------------+---------------+-------------+-------------+----------------+-------------+-------------+--------------+-------------+--------------------+--------------------+--------------------+----------+\n",
      "|months_loan_duration|amount|installment_rate|residence_history|age|existing_credits|default|dependents|checking_balance|credit_history|      purpose|savings_balance|employment_length|personal_status|other_debtors|     property|installment_plan|      housing|    telephone|foreign_worker|          job|            features|       rawPrediction|         probability|prediction|\n",
      "+--------------------+------+----------------+-----------------+---+----------------+-------+----------+----------------+--------------+-------------+---------------+-----------------+---------------+-------------+-------------+----------------+-------------+-------------+--------------+-------------+--------------------+--------------------+--------------------+----------+\n",
      "|                   4|  1544|               2|                1| 42|               3|      1|         2|   (3,[0],[1.0])| (4,[1],[1.0])|(9,[0],[1.0])|  (4,[0],[1.0])|    (4,[2],[1.0])|  (3,[0],[1.0])|(2,[0],[1.0])|(3,[1],[1.0])|   (2,[0],[1.0])|(2,[0],[1.0])|(1,[0],[1.0])| (1,[0],[1.0])|(3,[1],[1.0])|(48,[0,3,5,8,17,1...|[0.0,18.211093191...|[0.0,0.9105546595...|       1.0|\n",
      "|                   6|   362|               4|                4| 52|               2|      1|         1|   (3,[0],[1.0])| (4,[1],[1.0])|(9,[1],[1.0])|  (4,[2],[1.0])|    (4,[0],[1.0])|  (3,[1],[1.0])|(2,[0],[1.0])|(3,[0],[1.0])|   (2,[0],[1.0])|(2,[0],[1.0])|(1,[0],[1.0])| (1,[0],[1.0])|(3,[1],[1.0])|(48,[0,3,5,9,17,2...|[0.0,17.371032390...|[0.0,0.8685516195...|       1.0|\n",
      "|                   6|   454|               3|                1| 22|               1|      1|         1|   (3,[2],[1.0])| (4,[0],[1.0])|(9,[6],[1.0])|  (4,[0],[1.0])|    (4,[3],[1.0])|  (3,[2],[1.0])|(2,[0],[1.0])|(3,[2],[1.0])|   (2,[0],[1.0])|(2,[0],[1.0])|(1,[0],[1.0])| (1,[0],[1.0])|(3,[1],[1.0])|(48,[2,3,4,14,17,...|[0.0,13.743747374...|[0.0,0.6871873687...|       1.0|\n",
      "|                   6|   518|               3|                1| 29|               1|      1|         1|   (3,[0],[1.0])| (4,[0],[1.0])|(9,[0],[1.0])|  (4,[0],[1.0])|    (4,[0],[1.0])|  (3,[1],[1.0])|(2,[0],[1.0])|(3,[1],[1.0])|   (2,[0],[1.0])|(2,[0],[1.0])|(1,[0],[1.0])| (1,[0],[1.0])|(3,[0],[1.0])|(48,[0,3,4,8,17,1...|[0.0,17.285335835...|[0.0,0.8642667917...|       1.0|\n",
      "|                   6|   609|               4|                3| 37|               2|      1|         1|   (3,[1],[1.0])| (4,[1],[1.0])|(9,[1],[1.0])|  (4,[0],[1.0])|    (4,[2],[1.0])|  (3,[1],[1.0])|(2,[0],[1.0])|(3,[2],[1.0])|   (2,[0],[1.0])|(2,[0],[1.0])|(1,[0],[1.0])|     (1,[],[])|(3,[0],[1.0])|(48,[1,3,5,9,17,1...|[0.0,15.599321361...|[0.0,0.7799660680...|       1.0|\n",
      "+--------------------+------+----------------+-----------------+---+----------------+-------+----------+----------------+--------------+-------------+---------------+-----------------+---------------+-------------+-------------+----------------+-------------+-------------+--------------+-------------+--------------------+--------------------+--------------------+----------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_test_pred = forest_model.transform(df_test)\n",
    "df_test_pred.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confusion Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+---+\n",
      "|default|1.0|2.0|\n",
      "+-------+---+---+\n",
      "|      1|197| 14|\n",
      "|      2| 67| 18|\n",
      "+-------+---+---+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_test_pred.groupBy(\"default\").pivot(\"prediction\").count().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7263513513513513"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator = evaluation.MulticlassClassificationEvaluator(labelCol=\"default\", \n",
    "                                        metricName=\"accuracy\", predictionCol=\"prediction\")\n",
    "evaluator.evaluate(df_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of features:  48 \n",
      "Order of feature importance: \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>checking_balance_unknown</td>\n",
       "      <td>0.137719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>amount</td>\n",
       "      <td>0.108318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>months_loan_duration</td>\n",
       "      <td>0.107306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>checking_balance_&lt; 0 DM</td>\n",
       "      <td>0.084107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>credit_history_critical</td>\n",
       "      <td>0.050758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>age</td>\n",
       "      <td>0.037456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>savings_balance_unknown</td>\n",
       "      <td>0.034928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>checking_balance_1 - 200 DM</td>\n",
       "      <td>0.033914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>residence_history</td>\n",
       "      <td>0.028649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>purpose_car (new)</td>\n",
       "      <td>0.027793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>installment_plan_bank</td>\n",
       "      <td>0.024021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>personal_status_single male</td>\n",
       "      <td>0.020772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>property_real estate</td>\n",
       "      <td>0.019711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>housing_rent</td>\n",
       "      <td>0.019111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>existing_credits</td>\n",
       "      <td>0.017382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>installment_plan_none</td>\n",
       "      <td>0.016538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>installment_rate</td>\n",
       "      <td>0.016347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>credit_history_fully repaid this bank</td>\n",
       "      <td>0.015123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>savings_balance_101 - 500 DM</td>\n",
       "      <td>0.014589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>job_mangement self-employed</td>\n",
       "      <td>0.013875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>housing_own</td>\n",
       "      <td>0.012741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>employment_length_0 - 1 yrs</td>\n",
       "      <td>0.010733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>telephone_none</td>\n",
       "      <td>0.010121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>savings_balance_&lt; 100 DM</td>\n",
       "      <td>0.009821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>personal_status_female</td>\n",
       "      <td>0.009779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>savings_balance_501 - 1000 DM</td>\n",
       "      <td>0.009693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>other_debtors_guarantor</td>\n",
       "      <td>0.009417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>property_other</td>\n",
       "      <td>0.008785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>other_debtors_none</td>\n",
       "      <td>0.007644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>purpose_furniture</td>\n",
       "      <td>0.007480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>purpose_education</td>\n",
       "      <td>0.006903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>job_unskilled resident</td>\n",
       "      <td>0.006763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>employment_length_4 - 7 yrs</td>\n",
       "      <td>0.006574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>credit_history_repaid</td>\n",
       "      <td>0.005442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>job_skilled employee</td>\n",
       "      <td>0.004831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>credit_history_delayed</td>\n",
       "      <td>0.004603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>purpose_car (used)</td>\n",
       "      <td>0.004378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>purpose_others</td>\n",
       "      <td>0.004144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>employment_length_1 - 4 yrs</td>\n",
       "      <td>0.004059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>personal_status_married male</td>\n",
       "      <td>0.003893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>purpose_radio/tv</td>\n",
       "      <td>0.003618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>property_building society savings</td>\n",
       "      <td>0.003350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>purpose_business</td>\n",
       "      <td>0.003306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>dependents</td>\n",
       "      <td>0.003301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>purpose_repairs</td>\n",
       "      <td>0.002724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>employment_length_&gt; 7 yrs</td>\n",
       "      <td>0.002723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>foreign_worker_yes</td>\n",
       "      <td>0.002495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>purpose_domestic appliances</td>\n",
       "      <td>0.002262</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  feature  importance\n",
       "0                checking_balance_unknown    0.137719\n",
       "17                                 amount    0.108318\n",
       "3                    months_loan_duration    0.107306\n",
       "1                 checking_balance_< 0 DM    0.084107\n",
       "5                 credit_history_critical    0.050758\n",
       "36                                    age    0.037456\n",
       "19                savings_balance_unknown    0.034928\n",
       "2             checking_balance_1 - 200 DM    0.033914\n",
       "32                      residence_history    0.028649\n",
       "9                       purpose_car (new)    0.027793\n",
       "38                  installment_plan_bank    0.024021\n",
       "27            personal_status_single male    0.020772\n",
       "34                   property_real estate    0.019711\n",
       "40                           housing_rent    0.019111\n",
       "41                       existing_credits    0.017382\n",
       "37                  installment_plan_none    0.016538\n",
       "26                       installment_rate    0.016347\n",
       "7   credit_history_fully repaid this bank    0.015123\n",
       "20           savings_balance_101 - 500 DM    0.014589\n",
       "47            job_mangement self-employed    0.013875\n",
       "39                            housing_own    0.012741\n",
       "25            employment_length_0 - 1 yrs    0.010733\n",
       "43                         telephone_none    0.010121\n",
       "18               savings_balance_< 100 DM    0.009821\n",
       "28                 personal_status_female    0.009779\n",
       "21          savings_balance_501 - 1000 DM    0.009693\n",
       "31                other_debtors_guarantor    0.009417\n",
       "33                         property_other    0.008785\n",
       "30                     other_debtors_none    0.007644\n",
       "10                      purpose_furniture    0.007480\n",
       "13                      purpose_education    0.006903\n",
       "46                 job_unskilled resident    0.006763\n",
       "24            employment_length_4 - 7 yrs    0.006574\n",
       "4                   credit_history_repaid    0.005442\n",
       "45                   job_skilled employee    0.004831\n",
       "6                  credit_history_delayed    0.004603\n",
       "11                     purpose_car (used)    0.004378\n",
       "15                         purpose_others    0.004144\n",
       "22            employment_length_1 - 4 yrs    0.004059\n",
       "29           personal_status_married male    0.003893\n",
       "8                        purpose_radio/tv    0.003618\n",
       "35      property_building society savings    0.003350\n",
       "12                       purpose_business    0.003306\n",
       "42                             dependents    0.003301\n",
       "14                        purpose_repairs    0.002724\n",
       "23              employment_length_> 7 yrs    0.002723\n",
       "44                     foreign_worker_yes    0.002495\n",
       "16            purpose_domestic appliances    0.002262"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Total number of features: \", forest_model.numFeatures, \"\\nOrder of feature importance: \\n\")\n",
    "pd.DataFrame({\"importance\": forest_model.featureImportances.toArray(), \n",
    "              \"feature\": model.get_col_labels()\n",
    "             }).sort_values(\"importance\", ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building a pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.pipeline import Pipeline, PipelineModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stages: a list of pipeline stages (undefined)\n",
      "Accuracy 0.7601351351351351\n"
     ]
    }
   ],
   "source": [
    "credit = spark.read.options(header = True, inferSchema = True).csv(\"/data/credit-default.csv\").cache()\n",
    "\n",
    "label_col = \"default\"\n",
    "feature_cols = credit.columns\n",
    "feature_cols.remove(label_col)\n",
    "\n",
    "df_train, df_test = credit.randomSplit(weights=[0.7, 0.3], seed=1)\n",
    "\n",
    "\n",
    "pipeline = Pipeline()\n",
    "print(pipeline.explainParams())\n",
    "encoder = DFOneHotEncoder(label_column = label_col)\n",
    "vectorizer = VectorAssembler(inputCols = feature_cols, outputCol=\"features\")\n",
    "forest = RandomForestClassifier(labelCol=\"default\", featuresCol=\"features\", seed = 123)\n",
    "\n",
    "pipeline.setStages([encoder, vectorizer, forest])\n",
    "pipelineModel = pipeline.fit(df_train)\n",
    "df_test_pred = pipelineModel.transform(df_test)\n",
    "evaluator = evaluation.MulticlassClassificationEvaluator(labelCol=\"default\", \n",
    "                                        metricName=\"accuracy\", predictionCol=\"prediction\")\n",
    "\n",
    "accuracy = evaluator.evaluate(df_test_pred)\n",
    "print(\"Accuracy\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}