03-nan-data-validation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f94ec4d9-7f68-4573-851f-ec0e36f706a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "49c8ed69-9163-4e2e-b92a-60a3c6b2a1a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read correct schema from 02-notebook save a csv file \\\n",
    "# so that reading into pandas data frame can be possible \\\n",
    "# and also inferring schema.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "12315567-c944-4abc-9ff7-110bb89fbf71",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\"data/train_cleaned.csv\")\n",
    "test_df = pd.read_csv(\"data/test_cleaned.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bdf2347f-1703-4567-98d7-76b1678bb3c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education_num</th>\n",
       "      <th>marital_status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital_gain</th>\n",
       "      <th>capital_loss</th>\n",
       "      <th>hours_per_week</th>\n",
       "      <th>native_country</th>\n",
       "      <th>income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>77516</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>83311</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>215646</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>Private</td>\n",
       "      <td>234721</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>Private</td>\n",
       "      <td>338409</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Wife</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>Cuba</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32556</th>\n",
       "      <td>27</td>\n",
       "      <td>Private</td>\n",
       "      <td>257302</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>12</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Tech-support</td>\n",
       "      <td>Wife</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32557</th>\n",
       "      <td>40</td>\n",
       "      <td>Private</td>\n",
       "      <td>154374</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32558</th>\n",
       "      <td>58</td>\n",
       "      <td>Private</td>\n",
       "      <td>151910</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Widowed</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Unmarried</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32559</th>\n",
       "      <td>22</td>\n",
       "      <td>Private</td>\n",
       "      <td>201490</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32560</th>\n",
       "      <td>52</td>\n",
       "      <td>Self-emp-inc</td>\n",
       "      <td>287927</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Wife</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>15024</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>32561 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       age         workclass  fnlwgt   education  education_num  \\\n",
       "0       39         State-gov   77516   Bachelors             13   \n",
       "1       50  Self-emp-not-inc   83311   Bachelors             13   \n",
       "2       38           Private  215646     HS-grad              9   \n",
       "3       53           Private  234721        11th              7   \n",
       "4       28           Private  338409   Bachelors             13   \n",
       "...    ...               ...     ...         ...            ...   \n",
       "32556   27           Private  257302  Assoc-acdm             12   \n",
       "32557   40           Private  154374     HS-grad              9   \n",
       "32558   58           Private  151910     HS-grad              9   \n",
       "32559   22           Private  201490     HS-grad              9   \n",
       "32560   52      Self-emp-inc  287927     HS-grad              9   \n",
       "\n",
       "           marital_status         occupation   relationship   race     sex  \\\n",
       "0           Never-married       Adm-clerical  Not-in-family  White    Male   \n",
       "1      Married-civ-spouse    Exec-managerial        Husband  White    Male   \n",
       "2                Divorced  Handlers-cleaners  Not-in-family  White    Male   \n",
       "3      Married-civ-spouse  Handlers-cleaners        Husband  Black    Male   \n",
       "4      Married-civ-spouse     Prof-specialty           Wife  Black  Female   \n",
       "...                   ...                ...            ...    ...     ...   \n",
       "32556  Married-civ-spouse       Tech-support           Wife  White  Female   \n",
       "32557  Married-civ-spouse  Machine-op-inspct        Husband  White    Male   \n",
       "32558             Widowed       Adm-clerical      Unmarried  White  Female   \n",
       "32559       Never-married       Adm-clerical      Own-child  White    Male   \n",
       "32560  Married-civ-spouse    Exec-managerial           Wife  White  Female   \n",
       "\n",
       "       capital_gain  capital_loss  hours_per_week native_country  income  \n",
       "0              2174             0              40  United-States   <=50K  \n",
       "1                 0             0              13  United-States   <=50K  \n",
       "2                 0             0              40  United-States   <=50K  \n",
       "3                 0             0              40  United-States   <=50K  \n",
       "4                 0             0              40           Cuba   <=50K  \n",
       "...             ...           ...             ...            ...     ...  \n",
       "32556             0             0              38  United-States   <=50K  \n",
       "32557             0             0              40  United-States    >50K  \n",
       "32558             0             0              40  United-States   <=50K  \n",
       "32559             0             0              20  United-States   <=50K  \n",
       "32560         15024             0              40  United-States    >50K  \n",
       "\n",
       "[32561 rows x 15 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "add40649-b517-445a-90a2-a91656604b89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education_num</th>\n",
       "      <th>marital_status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital_gain</th>\n",
       "      <th>capital_loss</th>\n",
       "      <th>hours_per_week</th>\n",
       "      <th>native_country</th>\n",
       "      <th>income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25</td>\n",
       "      <td>Private</td>\n",
       "      <td>226802</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>89814</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Farming-fishing</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>336951</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>12</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Protective-serv</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44</td>\n",
       "      <td>Private</td>\n",
       "      <td>160323</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>7688</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18</td>\n",
       "      <td>?</td>\n",
       "      <td>103497</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>?</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16276</th>\n",
       "      <td>39</td>\n",
       "      <td>Private</td>\n",
       "      <td>215419</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16277</th>\n",
       "      <td>64</td>\n",
       "      <td>?</td>\n",
       "      <td>321403</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Widowed</td>\n",
       "      <td>?</td>\n",
       "      <td>Other-relative</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16278</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>374983</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16279</th>\n",
       "      <td>44</td>\n",
       "      <td>Private</td>\n",
       "      <td>83891</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>Asian-Pac-Islander</td>\n",
       "      <td>Male</td>\n",
       "      <td>5455</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16280</th>\n",
       "      <td>35</td>\n",
       "      <td>Self-emp-inc</td>\n",
       "      <td>182148</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16281 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       age     workclass  fnlwgt     education  education_num  \\\n",
       "0       25       Private  226802          11th              7   \n",
       "1       38       Private   89814       HS-grad              9   \n",
       "2       28     Local-gov  336951    Assoc-acdm             12   \n",
       "3       44       Private  160323  Some-college             10   \n",
       "4       18             ?  103497  Some-college             10   \n",
       "...    ...           ...     ...           ...            ...   \n",
       "16276   39       Private  215419     Bachelors             13   \n",
       "16277   64             ?  321403       HS-grad              9   \n",
       "16278   38       Private  374983     Bachelors             13   \n",
       "16279   44       Private   83891     Bachelors             13   \n",
       "16280   35  Self-emp-inc  182148     Bachelors             13   \n",
       "\n",
       "           marital_status         occupation    relationship  \\\n",
       "0           Never-married  Machine-op-inspct       Own-child   \n",
       "1      Married-civ-spouse    Farming-fishing         Husband   \n",
       "2      Married-civ-spouse    Protective-serv         Husband   \n",
       "3      Married-civ-spouse  Machine-op-inspct         Husband   \n",
       "4           Never-married                  ?       Own-child   \n",
       "...                   ...                ...             ...   \n",
       "16276            Divorced     Prof-specialty   Not-in-family   \n",
       "16277             Widowed                  ?  Other-relative   \n",
       "16278  Married-civ-spouse     Prof-specialty         Husband   \n",
       "16279            Divorced       Adm-clerical       Own-child   \n",
       "16280  Married-civ-spouse    Exec-managerial         Husband   \n",
       "\n",
       "                     race     sex  capital_gain  capital_loss  hours_per_week  \\\n",
       "0                   Black    Male             0             0              40   \n",
       "1                   White    Male             0             0              50   \n",
       "2                   White    Male             0             0              40   \n",
       "3                   Black    Male          7688             0              40   \n",
       "4                   White  Female             0             0              30   \n",
       "...                   ...     ...           ...           ...             ...   \n",
       "16276               White  Female             0             0              36   \n",
       "16277               Black    Male             0             0              40   \n",
       "16278               White    Male             0             0              50   \n",
       "16279  Asian-Pac-Islander    Male          5455             0              40   \n",
       "16280               White    Male             0             0              60   \n",
       "\n",
       "      native_country   income  \n",
       "0      United-States   <=50K.  \n",
       "1      United-States   <=50K.  \n",
       "2      United-States    >50K.  \n",
       "3      United-States    >50K.  \n",
       "4      United-States   <=50K.  \n",
       "...              ...      ...  \n",
       "16276  United-States   <=50K.  \n",
       "16277  United-States   <=50K.  \n",
       "16278  United-States   <=50K.  \n",
       "16279  United-States   <=50K.  \n",
       "16280  United-States    >50K.  \n",
       "\n",
       "[16281 rows x 15 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4968a0df-acba-45e0-8484-3e5dc0e949d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import read_yaml, create_directories\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3a757cec-3416-45b3-b27e-8bc1503310f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yaml file: config.yaml loaded successfully\n",
      "yaml file: schema.yaml loaded successfully\n"
     ]
    }
   ],
   "source": [
    "config_box = read_yaml(\"config.yaml\")\n",
    "schema_box = read_yaml(\"schema.yaml\")\n",
    "config = config_box.data_validation\n",
    "schema = schema_box.FEATURES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fc89a709-bc47-4735-b9d2-7a338c773a53",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class DataValidation:\n",
    "    def __init__(self, config, schema):\n",
    "        self.config = config\n",
    "        self.schema = schema\n",
    "\n",
    "    def validate_columns(self):\n",
    "        validation_status = None\n",
    "        \n",
    "        data = pd.read_csv(self.config.train_data_dir)\n",
    "        all_columns = list(data.columns)\n",
    "        schema = self.schema.keys()\n",
    "        schema_values = self.schema.values()\n",
    "        for column in all_columns:\n",
    "            if column not in schema:\n",
    "                validation_status = False\n",
    "                with open(self.config.STATUS_FILE, 'w') as file:\n",
    "                    file.write(f\"validation status: {validation_status}\")\n",
    "                \n",
    "            else:\n",
    "                validation_status = True\n",
    "                with open(self.config.STATUS_FILE, 'w') as file:\n",
    "                    file.write(f\"validation status: {validation_status}\")\n",
    "        return validation_status  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1e80daf7-d7d0-46fc-9288-f9b8122f7461",
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_data = DataValidation(config, schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7a84094d-ca53-4e29-adbb-b9826a07b1b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_data.validate_columns()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9e4543bf-ca51-45eb-be53-d1e79f9ceaa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.dtypes.Int64DType"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cdfc4ce-9c96-455c-8244-1216bf313000",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}