Data_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "abc670c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ucimlrepo import fetch_ucirepo \n",
    "from collections import Counter\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.mixture import GaussianMixture\n",
    "from sklearn.metrics import silhouette_samples, silhouette_score\n",
    "import matplotlib.cm as cm\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from scipy.spatial.distance import cdist\n",
    "import time\n",
    "import seaborn as sns\n",
    "from matplotlib.patches import Ellipse\n",
    "from scipy import linalg\n",
    "from scipy import interpolate\n",
    "from scipy.signal import argrelextrema\n",
    "from kneed import KneeLocator\n",
    "import itertools\n",
    "import matplotlib as mpl\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from matplotlib import rcParams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "746bbab3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wine Quality Data\n",
    "# fetch dataset \n",
    "wq_df = fetch_ucirepo(id=186) \n",
    "  \n",
    "# data (as pandas dataframes) \n",
    "wq_X = wq_df.data.features \n",
    "wq_y = wq_df.data.targets \n",
    "\n",
    "# Breast Cancer Data\n",
    "# fetch dataset \n",
    "bc_df = fetch_ucirepo(id=17) \n",
    "  \n",
    "# data (as pandas dataframes) \n",
    "bc_X = bc_df.data.features \n",
    "bc_y = bc_df.data.targets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d1ccd876",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'freqs' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m wq_freqs \u001b[38;5;241m=\u001b[39mwq_y\u001b[38;5;241m.\u001b[39mvalue_counts() \u001b[38;5;241m/\u001b[39m\u001b[38;5;28mlen\u001b[39m(wq_y)\n\u001b[0;32m----> 2\u001b[0m wq_std_errors \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39msqrt(\u001b[43mfreqs\u001b[49m\u001b[38;5;241m*\u001b[39m(\u001b[38;5;241m1\u001b[39m\u001b[38;5;241m-\u001b[39mfreqs)\u001b[38;5;241m/\u001b[39m\u001b[38;5;28mlen\u001b[39m(wq_y))\n\u001b[1;32m      3\u001b[0m wq_expected_frequency \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\u001b[38;5;241m/\u001b[39m\u001b[38;5;28mlen\u001b[39m(np\u001b[38;5;241m.\u001b[39munique(wq_y))\n\u001b[1;32m      5\u001b[0m wq_freqs\u001b[38;5;241m.\u001b[39mplot(kind\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbar\u001b[39m\u001b[38;5;124m'\u001b[39m,yerr\u001b[38;5;241m=\u001b[39mwq_std_errors\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m1.96\u001b[39m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'freqs' is not defined"
     ]
    }
   ],
   "source": [
    "wq_freqs =wq_y.value_counts() /len(wq_y)\n",
    "wq_std_errors = np.sqrt(freqs*(1-freqs)/len(wq_y))\n",
    "wq_expected_frequency = 1/len(np.unique(wq_y))\n",
    "\n",
    "wq_freqs.plot(kind='bar',yerr=wq_std_errors*1.96)\n",
    "plt.title('Wine Quality Distribution')\n",
    "plt.axhline(wq_expected_frequency,color='red',linestyle='--')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bb1a3fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "bc_freqs =bc_y.value_counts() /len(bc_y)\n",
    "bc_std_errors = np.sqrt(freqs*(1-freqs)/len(bc_y))\n",
    "bc_expected_frequency = 1/len(np.unique(bc_y))\n",
    "\n",
    "bc_freqs.plot(kind='bar',yerr=bc_std_errors*1.96)\n",
    "plt.title('Breast Cancer Distribution')\n",
    "plt.axhline(bc_expected_frequency,color='red',linestyle='--')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cdfa9fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ((ax1, ax2)) = plt.subplots(1, 2, figsize=(18, 6))\n",
    "rcParams['font.weight'] = 'bold'\n",
    "\n",
    "wq_freqs.plot(kind='bar',yerr=wq_std_errors*1.96, ax=ax1)\n",
    "ax1.set_title('Wine Quality Distribution')\n",
    "ax1.axhline(wq_expected_frequency,color='red',linestyle='--')\n",
    "\n",
    "bc_freqs.plot(kind='bar',yerr=bc_std_errors*1.96, ax=ax2)\n",
    "ax2.set_title('Breast Cancer Distribution')\n",
    "ax2.axhline(bc_expected_frequency,color='red',linestyle='--')\n",
    "plt.savefig(f'plots/distribution.png', dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d62ac82",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}