MLFinalClassification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas import read_excel, DataFrame, IndexSlice\n",
    "from random import choices\n",
    "from sklearn.preprocessing import LabelBinarizer, normalize\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">Q1</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Q2</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Q3</th>\n",
       "      <th>Q4</th>\n",
       "      <th>...</th>\n",
       "      <th>Q12</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Q13</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Q14</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Q15</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Yes</th>\n",
       "      <th>No</th>\n",
       "      <th>Undecided</th>\n",
       "      <th>Yes</th>\n",
       "      <th>No</th>\n",
       "      <th>Undecided</th>\n",
       "      <th>Yes</th>\n",
       "      <th>No</th>\n",
       "      <th>Undecided</th>\n",
       "      <th>Yes</th>\n",
       "      <th>...</th>\n",
       "      <th>Undecided</th>\n",
       "      <th>Yes</th>\n",
       "      <th>No</th>\n",
       "      <th>Undecided</th>\n",
       "      <th>Yes</th>\n",
       "      <th>No</th>\n",
       "      <th>Undecided</th>\n",
       "      <th>Yes</th>\n",
       "      <th>No</th>\n",
       "      <th>Undecided</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>94</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>89</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>89</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>56</td>\n",
       "      <td>38</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>72</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>72</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>56</td>\n",
       "      <td>44</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>83</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>94</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>89</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>55</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>28</td>\n",
       "      <td>72</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>44</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>78</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Q1                Q2                 Q3                 Q4  ...       Q12  \\\n",
       "  Yes  No Undecided Yes   No Undecided Yes   No Undecided Yes  ... Undecided   \n",
       "0  94   6         0  11   89         0   0  100         0  11  ...         0   \n",
       "1  72  28         0  28   72         0  46   54         0  61  ...         0   \n",
       "2  89  11         0  17   83         0   6   94         0  50  ...         0   \n",
       "3  28  72         0   0  100         0  94    6         0   6  ...         0   \n",
       "\n",
       "  Q13                Q14               Q15                \n",
       "  Yes  No Undecided  Yes  No Undecided Yes  No Undecided  \n",
       "0  89  11         0  100   0         0  56  38         6  \n",
       "1  61  39         0   50  50         0  56  44         0  \n",
       "2  89  11         0   94   6         0  28  55        17  \n",
       "3  94   6         0   50  44         6   0  78        22  \n",
       "\n",
       "[4 rows x 45 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = 'data/Take home exam dataset.xlsx'\n",
    "df_groups = read_excel(data_path, header=[0,1])\n",
    "df_groups.drop(['Unnamed: 0_level_0', 'Unnamed: 1_level_0'], axis=1, inplace=True, level=0)\n",
    "headers = ['GROUP', 'Q1', 'Q2', 'Q3', 'Q4', 'Q5', 'Q6', 'Q7', 'Q8', 'Q9', 'Q10', 'Q11', 'Q12', 'Q13', 'Q14', 'Q15']\n",
    "df_groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def standardize_dataset(df_groups, possible_answers=[1, 0, 0.5]):\n",
    "    num_people = 20\n",
    "    people = []\n",
    "    headers = ['Q1', 'Q2', 'Q3', 'Q4', 'Q5', 'Q6', 'Q7', 'Q8', 'Q9', 'Q10', 'Q11', 'Q12', 'Q13', 'Q14', 'Q15']\n",
    "    for group_num, group in enumerate(['g1', 'g2', 'g3', 'g4']):\n",
    "        for person in range(num_people):\n",
    "            answers = [group]\n",
    "            for question in headers:\n",
    "                answer_distributions = df_groups[question]\n",
    "                weights = [\n",
    "                    answer_distributions.Yes[group_num]/100,\n",
    "                    answer_distributions.No[group_num]/100,\n",
    "                    answer_distributions.Undecided[group_num]/100\n",
    "                ]\n",
    "                answers.extend(choices(possible_answers, weights))\n",
    "            people.append(answers)\n",
    "    df_groups.columns.levels[0][:-2].sort_values()\n",
    "    return people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GROUP</th>\n",
       "      <th>Q1</th>\n",
       "      <th>Q2</th>\n",
       "      <th>Q3</th>\n",
       "      <th>Q4</th>\n",
       "      <th>Q5</th>\n",
       "      <th>Q6</th>\n",
       "      <th>Q7</th>\n",
       "      <th>Q8</th>\n",
       "      <th>Q9</th>\n",
       "      <th>Q10</th>\n",
       "      <th>Q11</th>\n",
       "      <th>Q12</th>\n",
       "      <th>Q13</th>\n",
       "      <th>Q14</th>\n",
       "      <th>Q15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>g1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>g1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>g1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>g1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>g1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>g4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>g4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>g4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>g4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>g4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   GROUP  Q1  Q2  Q3   Q4   Q5  Q6  Q7  Q8  Q9  Q10  Q11  Q12  Q13  Q14  Q15\n",
       "0     g1   1   0   0  0.5  0.5   0   0   0   0    0    1    1    0  1.0  1.0\n",
       "1     g1   1   0   0  0.5  0.5   0   0   0   1    0    1    1    1  1.0  0.0\n",
       "2     g1   1   0   0  0.0  1.0   0   0   0   0    0    1    1    1  1.0  1.0\n",
       "3     g1   1   0   0  0.5  0.0   1   0   0   1    0    1    1    1  1.0  1.0\n",
       "4     g1   1   1   0  0.5  0.5   0   0   0   0    0    1    1    1  1.0  0.0\n",
       "..   ...  ..  ..  ..  ...  ...  ..  ..  ..  ..  ...  ...  ...  ...  ...  ...\n",
       "75    g4   1   0   1  0.0  1.0   1   0   0   1    0    1    1    1  0.5  0.0\n",
       "76    g4   0   0   1  0.0  1.0   1   0   0   1    0    1    1    1  0.0  0.0\n",
       "77    g4   0   0   1  0.0  1.0   1   0   0   1    0    1    1    1  1.0  0.0\n",
       "78    g4   1   0   1  0.0  1.0   1   0   0   1    0    1    1    1  1.0  0.0\n",
       "79    g4   1   0   1  0.0  1.0   1   0   0   1    0    0    1    1  1.0  0.5\n",
       "\n",
       "[80 rows x 16 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people = standardize_dataset(df_groups)\n",
    "df = DataFrame(people, columns=headers)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.42640143, 0.        , 0.        , ..., 0.        , 0.42640143,\n",
       "        0.42640143],\n",
       "       [0.39223227, 0.        , 0.        , ..., 0.39223227, 0.39223227,\n",
       "        0.        ],\n",
       "       [0.37796447, 0.        , 0.        , ..., 0.37796447, 0.37796447,\n",
       "        0.37796447],\n",
       "       ...,\n",
       "       [0.        , 0.        , 0.35355339, ..., 0.35355339, 0.35355339,\n",
       "        0.        ],\n",
       "       [0.33333333, 0.        , 0.33333333, ..., 0.33333333, 0.33333333,\n",
       "        0.        ],\n",
       "       [0.34815531, 0.        , 0.34815531, ..., 0.34815531, 0.34815531,\n",
       "        0.17407766]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# One Hot Encoding the Group Categories\n",
    "group_lb = LabelBinarizer()\n",
    "Y = group_lb.fit_transform(df.GROUP.values)\n",
    "\n",
    "# Normalize the X input\n",
    "FEATURES = df.columns[1:]\n",
    "X_data = df[FEATURES].values\n",
    "X_data = normalize(X_data)\n",
    "X_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(56, 15)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data Split into 70% training 30% testing\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_data, Y, test_size=0.3)#, random_state=1)\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifier 1 - Artificial Neural Network Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\users\\animu\\git\\ontariotech\\machine-learning\\sofe4620-final\\env\\lib\\site-packages\\tensorflow\\python\\compat\\v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "non-resource variables are not supported in the long term\n"
     ]
    }
   ],
   "source": [
    "import tensorflow.compat.v1 as tf\n",
    "from numpy import mean, argmax\n",
    "tf.disable_v2_behavior()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.01\n",
    "training_epochs = 100\n",
    "\n",
    "# Neural Network Parameters\n",
    "n_hidden_1 = 20\n",
    "n_hidden_2 = 16\n",
    "\n",
    "n_input = X_train.shape[1] # input shape (105, 4)\n",
    "n_classes = y_train.shape[1] # classes to predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inputs\n",
    "X = tf.placeholder(\"float\", shape=[None, n_input])\n",
    "y = tf.placeholder(\"float\", shape=[None, n_classes])\n",
    "# Dictionary of Weights and Biases\n",
    "weights = {\n",
    "  'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),\n",
    "  'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),\n",
    "  'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))\n",
    "}\n",
    "biases = {\n",
    "  'b1': tf.Variable(tf.random_normal([n_hidden_1])),\n",
    "  'b2': tf.Variable(tf.random_normal([n_hidden_2])),\n",
    "  'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model Forward Propagation step\n",
    "def forward_propagation(x):\n",
    "    # Hidden layer1\n",
    "    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])\n",
    "    layer_1 = tf.nn.relu(layer_1)\n",
    "    \n",
    "    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])\n",
    "    layer_2 = tf.nn.relu(layer_2)\n",
    "    # Output fully connected layer\n",
    "    out_layer = tf.matmul(layer_2, weights['out']) + biases['out'] \n",
    "    return out_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model Outputs\n",
    "yhat = forward_propagation(X)\n",
    "ypredict = tf.argmax(yhat, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-12-303a27d0f28f>:2: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Back Propagation\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=yhat))\n",
    "# optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "train_op = optimizer.minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch = 1, train accuracy = 39.29, test accuracy = 33.33\n",
      "Epoch = 2, train accuracy = 55.36, test accuracy = 45.83\n",
      "Epoch = 3, train accuracy = 69.64, test accuracy = 58.33\n",
      "Epoch = 4, train accuracy = 73.21, test accuracy = 62.50\n",
      "Epoch = 5, train accuracy = 80.36, test accuracy = 75.00\n",
      "Epoch = 6, train accuracy = 87.50, test accuracy = 79.17\n",
      "Epoch = 7, train accuracy = 87.50, test accuracy = 79.17\n",
      "Epoch = 8, train accuracy = 91.07, test accuracy = 79.17\n",
      "Epoch = 9, train accuracy = 91.07, test accuracy = 79.17\n",
      "Epoch = 10, train accuracy = 92.86, test accuracy = 79.17\n",
      "Epoch = 11, train accuracy = 94.64, test accuracy = 75.00\n",
      "Epoch = 12, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 13, train accuracy = 94.64, test accuracy = 75.00\n",
      "Epoch = 14, train accuracy = 96.43, test accuracy = 75.00\n",
      "Epoch = 15, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 16, train accuracy = 96.43, test accuracy = 75.00\n",
      "Epoch = 17, train accuracy = 94.64, test accuracy = 75.00\n",
      "Epoch = 18, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 19, train accuracy = 96.43, test accuracy = 75.00\n",
      "Epoch = 20, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 21, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 22, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 23, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 24, train accuracy = 96.43, test accuracy = 70.83\n",
      "Epoch = 25, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 26, train accuracy = 96.43, test accuracy = 75.00\n",
      "Epoch = 27, train accuracy = 96.43, test accuracy = 75.00\n",
      "Epoch = 28, train accuracy = 96.43, test accuracy = 70.83\n",
      "Epoch = 29, train accuracy = 96.43, test accuracy = 66.67\n",
      "Epoch = 30, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 31, train accuracy = 96.43, test accuracy = 62.50\n",
      "Epoch = 32, train accuracy = 96.43, test accuracy = 70.83\n",
      "Epoch = 33, train accuracy = 96.43, test accuracy = 66.67\n",
      "Epoch = 34, train accuracy = 94.64, test accuracy = 62.50\n",
      "Epoch = 35, train accuracy = 91.07, test accuracy = 62.50\n",
      "Epoch = 36, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 37, train accuracy = 92.86, test accuracy = 66.67\n",
      "Epoch = 38, train accuracy = 96.43, test accuracy = 66.67\n",
      "Epoch = 39, train accuracy = 92.86, test accuracy = 62.50\n",
      "Epoch = 40, train accuracy = 85.71, test accuracy = 62.50\n",
      "Epoch = 41, train accuracy = 92.86, test accuracy = 62.50\n",
      "Epoch = 42, train accuracy = 91.07, test accuracy = 62.50\n",
      "Epoch = 43, train accuracy = 92.86, test accuracy = 66.67\n",
      "Epoch = 44, train accuracy = 92.86, test accuracy = 66.67\n",
      "Epoch = 45, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 46, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 47, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 48, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 49, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 50, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 51, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 52, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 53, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 54, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 55, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 56, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 57, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 58, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 59, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 60, train accuracy = 94.64, test accuracy = 62.50\n",
      "Epoch = 61, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 62, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 63, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 64, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 65, train accuracy = 94.64, test accuracy = 75.00\n",
      "Epoch = 66, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 67, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 68, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 69, train accuracy = 94.64, test accuracy = 75.00\n",
      "Epoch = 70, train accuracy = 94.64, test accuracy = 75.00\n",
      "Epoch = 71, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 72, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 73, train accuracy = 94.64, test accuracy = 75.00\n",
      "Epoch = 74, train accuracy = 94.64, test accuracy = 79.17\n",
      "Epoch = 75, train accuracy = 91.07, test accuracy = 79.17\n",
      "Epoch = 76, train accuracy = 94.64, test accuracy = 79.17\n",
      "Epoch = 77, train accuracy = 92.86, test accuracy = 79.17\n",
      "Epoch = 78, train accuracy = 96.43, test accuracy = 87.50\n",
      "Epoch = 79, train accuracy = 96.43, test accuracy = 87.50\n",
      "Epoch = 80, train accuracy = 96.43, test accuracy = 75.00\n",
      "Epoch = 81, train accuracy = 94.64, test accuracy = 66.67\n",
      "Epoch = 82, train accuracy = 94.64, test accuracy = 75.00\n",
      "Epoch = 83, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 84, train accuracy = 96.43, test accuracy = 70.83\n",
      "Epoch = 85, train accuracy = 96.43, test accuracy = 70.83\n",
      "Epoch = 86, train accuracy = 96.43, test accuracy = 70.83\n",
      "Epoch = 87, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 88, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 89, train accuracy = 96.43, test accuracy = 70.83\n",
      "Epoch = 90, train accuracy = 94.64, test accuracy = 70.83\n",
      "Epoch = 91, train accuracy = 96.43, test accuracy = 70.83\n",
      "Epoch = 92, train accuracy = 96.43, test accuracy = 75.00\n",
      "Epoch = 93, train accuracy = 96.43, test accuracy = 75.00\n",
      "Epoch = 94, train accuracy = 96.43, test accuracy = 87.50\n",
      "Epoch = 95, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 96, train accuracy = 96.43, test accuracy = 79.17\n",
      "Epoch = 97, train accuracy = 96.43, test accuracy = 75.00\n",
      "Epoch = 98, train accuracy = 96.43, test accuracy = 87.50\n",
      "Epoch = 99, train accuracy = 96.43, test accuracy = 83.33\n",
      "Epoch = 100, train accuracy = 96.43, test accuracy = 87.50\n",
      "Time taken: 0:00:05.371999\n"
     ]
    }
   ],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "from datetime import datetime\n",
    "startTime = datetime.now()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    # Running the model through epochs\n",
    "    for epoch in range(training_epochs):\n",
    "        for i in range(len(X_train)):\n",
    "            summary = sess.run(train_op, feed_dict={X: X_train[i: i + 1], y: y_train[i: i + 1]})\n",
    "        \n",
    "        train_accuracy = mean(argmax(y_train, axis=1) == sess.run(ypredict, feed_dict={X: X_train, y: y_train}))\n",
    "        test_accuracy  = mean(argmax(y_test, axis=1) == sess.run(ypredict, feed_dict={X: X_test, y: y_test}))\n",
    "                \n",
    "        print(f\"Epoch = {epoch + 1}, train accuracy = {100. * train_accuracy:.2f}, test accuracy = {100. * test_accuracy:.2f}\")\n",
    "    sess.close()\n",
    "print(\"Time taken:\", datetime.now() - startTime)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifier 2 - Statistical Probabilistic Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GROUP</th>\n",
       "      <th>Q1</th>\n",
       "      <th>Q2</th>\n",
       "      <th>Q3</th>\n",
       "      <th>Q4</th>\n",
       "      <th>Q5</th>\n",
       "      <th>Q6</th>\n",
       "      <th>Q7</th>\n",
       "      <th>Q8</th>\n",
       "      <th>Q9</th>\n",
       "      <th>Q10</th>\n",
       "      <th>Q11</th>\n",
       "      <th>Q12</th>\n",
       "      <th>Q13</th>\n",
       "      <th>Q14</th>\n",
       "      <th>Q15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>g1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Undecided</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>g1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Undecided</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>g1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Undecided</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>g1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>g1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>g4</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>g4</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>g4</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>g4</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Undecided</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>g4</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   GROUP   Q1   Q2   Q3         Q4         Q5   Q6  Q7  Q8   Q9 Q10  Q11  Q12  \\\n",
       "0     g1  Yes   No   No  Undecided         No   No  No  No   No  No  Yes  Yes   \n",
       "1     g1  Yes   No   No  Undecided         No  Yes  No  No  Yes  No  Yes  Yes   \n",
       "2     g1  Yes   No   No         No  Undecided   No  No  No  Yes  No  Yes  Yes   \n",
       "3     g1  Yes   No   No        Yes        Yes   No  No  No  Yes  No  Yes  Yes   \n",
       "4     g1  Yes  Yes   No         No        Yes  Yes  No  No  Yes  No  Yes  Yes   \n",
       "..   ...  ...  ...  ...        ...        ...  ...  ..  ..  ...  ..  ...  ...   \n",
       "75    g4   No   No  Yes         No        Yes  Yes  No  No  Yes  No  Yes  Yes   \n",
       "76    g4   No   No  Yes        Yes        Yes  Yes  No  No  Yes  No  Yes  Yes   \n",
       "77    g4   No   No  Yes         No        Yes  Yes  No  No  Yes  No  Yes  Yes   \n",
       "78    g4   No   No  Yes         No        Yes  Yes  No  No  Yes  No  Yes  Yes   \n",
       "79    g4   No   No  Yes         No        Yes   No  No  No  Yes  No  Yes  Yes   \n",
       "\n",
       "    Q13  Q14        Q15  \n",
       "0   Yes  Yes        Yes  \n",
       "1   Yes  Yes         No  \n",
       "2   Yes  Yes         No  \n",
       "3   Yes  Yes        Yes  \n",
       "4   Yes  Yes        Yes  \n",
       "..  ...  ...        ...  \n",
       "75  Yes  Yes         No  \n",
       "76  Yes   No         No  \n",
       "77  Yes   No         No  \n",
       "78  Yes   No  Undecided  \n",
       "79  Yes   No         No  \n",
       "\n",
       "[80 rows x 16 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people = standardize_dataset(df_groups, possible_answers=['Yes', 'No', 'Undecided'])\n",
    "df = DataFrame(people, columns=headers)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Yes', 'No', 'No', ..., 'Yes', 'Yes', 'Yes'],\n",
       "       ['Yes', 'No', 'No', ..., 'Yes', 'Yes', 'No'],\n",
       "       ['Yes', 'No', 'No', ..., 'Yes', 'Yes', 'No'],\n",
       "       ...,\n",
       "       ['No', 'No', 'Yes', ..., 'Yes', 'No', 'No'],\n",
       "       ['No', 'No', 'Yes', ..., 'Yes', 'No', 'Undecided'],\n",
       "       ['No', 'No', 'Yes', ..., 'Yes', 'No', 'No']], dtype=object)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y = df.GROUP.values\n",
    "\n",
    "# Normalize the X input\n",
    "FEATURES = df.columns[1:]\n",
    "X_data = df[FEATURES].values\n",
    "X_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groups = ['g1', 'g2', 'g3', 'g4']\n",
    "correct = 0\n",
    "for person, group in zip(X_data, Y):\n",
    "    scores = [0, 0, 0, 0]\n",
    "    for i, question in enumerate(headers[1:]):\n",
    "        answer_given = person[i]\n",
    "        for j, probability in enumerate(df_groups[question][answer_given]):\n",
    "            scores[j] += probability\n",
    "    if groups[scores.index(max(scores))] == group:\n",
    "        correct += 1\n",
    "accuracy = correct / len(X_data)\n",
    "accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}