emnist-train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3843691b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-26T23:41:23.321959Z",
     "iopub.status.busy": "2024-12-26T23:41:23.321548Z",
     "iopub.status.idle": "2024-12-26T23:41:23.741877Z",
     "shell.execute_reply": "2024-12-26T23:41:23.740922Z"
    },
    "papermill": {
     "duration": 0.426957,
     "end_time": "2024-12-26T23:41:23.743871",
     "exception": false,
     "start_time": "2024-12-26T23:41:23.316914",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import struct\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "import joblib\n",
    "import struct\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.svm import SVC\n",
    "import time\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, Input\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from sklearn.preprocessing import StandardScaler\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "166165b7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-26T23:41:23.752133Z",
     "iopub.status.busy": "2024-12-26T23:41:23.751490Z",
     "iopub.status.idle": "2024-12-26T23:41:31.004530Z",
     "shell.execute_reply": "2024-12-26T23:41:31.003372Z"
    },
    "papermill": {
     "duration": 7.259226,
     "end_time": "2024-12-26T23:41:31.006435",
     "exception": false,
     "start_time": "2024-12-26T23:41:23.747209",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "images = np.load(\"/kaggle/input/emnist-preprocess/images.npy\")\n",
    "labels = np.load(\"/kaggle/input/emnist-preprocess/labels.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84982f60",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-26T23:41:31.017272Z",
     "iopub.status.busy": "2024-12-26T23:41:31.016748Z",
     "iopub.status.idle": "2024-12-26T23:41:31.023146Z",
     "shell.execute_reply": "2024-12-26T23:41:31.021859Z"
    },
    "papermill": {
     "duration": 0.015011,
     "end_time": "2024-12-26T23:41:31.026165",
     "exception": false,
     "start_time": "2024-12-26T23:41:31.011154",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(images.shape)\n",
    "print(labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5248b2a9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-26T23:41:31.034866Z",
     "iopub.status.busy": "2024-12-26T23:41:31.034372Z",
     "iopub.status.idle": "2024-12-26T23:41:40.142436Z",
     "shell.execute_reply": "2024-12-26T23:41:40.141540Z"
    },
    "papermill": {
     "duration": 9.113986,
     "end_time": "2024-12-26T23:41:40.144247",
     "exception": false,
     "start_time": "2024-12-26T23:41:31.030261",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# One hot encoding\n",
    "labels = to_categorical(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0ddd3ef9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-26T23:41:41.037743Z",
     "iopub.status.busy": "2024-12-26T23:41:41.037004Z",
     "iopub.status.idle": "2024-12-26T23:41:41.751764Z",
     "shell.execute_reply": "2024-12-26T23:41:41.750706Z"
    },
    "papermill": {
     "duration": 0.720986,
     "end_time": "2024-12-26T23:41:41.753681",
     "exception": false,
     "start_time": "2024-12-26T23:41:41.032695",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_x,test_x,train_y,test_y = train_test_split(images,labels,test_size=0.2,random_state = 42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e5a143a",
   "metadata": {
    "papermill": {
     "duration": 0.002824,
     "end_time": "2024-12-26T23:41:41.759878",
     "exception": false,
     "start_time": "2024-12-26T23:41:41.757054",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Model setup"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff1194e8",
   "metadata": {
    "papermill": {
     "duration": 0.002656,
     "end_time": "2024-12-26T23:41:41.765640",
     "exception": false,
     "start_time": "2024-12-26T23:41:41.762984",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "89342e3f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-26T23:41:41.790386Z",
     "iopub.status.busy": "2024-12-26T23:41:41.790064Z",
     "iopub.status.idle": "2024-12-26T23:41:41.935776Z",
     "shell.execute_reply": "2024-12-26T23:41:41.934798Z"
    },
    "papermill": {
     "duration": 0.151481,
     "end_time": "2024-12-26T23:41:41.937600",
     "exception": false,
     "start_time": "2024-12-26T23:41:41.786119",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cnn_model = Sequential()\n",
    "cnn_model.add(Input(shape=(28, 28, 1)))\n",
    "cnn_model.add(Conv2D(32, (3, 3), activation='relu'))\n",
    "cnn_model.add(MaxPooling2D((2, 2)))\n",
    "cnn_model.add(Flatten())\n",
    "cnn_model.add(Dense(512, activation='relu'))\n",
    "cnn_model.add(Dense(128, activation='relu'))\n",
    "cnn_model.add(Dense(46, activation='softmax'))\n",
    "cnn_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac66ec2d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-26T23:41:41.945296Z",
     "iopub.status.busy": "2024-12-26T23:41:41.944984Z",
     "iopub.status.idle": "2024-12-26T23:41:41.963725Z",
     "shell.execute_reply": "2024-12-26T23:41:41.962947Z"
    },
    "papermill": {
     "duration": 0.024078,
     "end_time": "2024-12-26T23:41:41.965169",
     "exception": false,
     "start_time": "2024-12-26T23:41:41.941091",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd22ec2a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-26T23:41:41.973828Z",
     "iopub.status.busy": "2024-12-26T23:41:41.973498Z",
     "iopub.status.idle": "2024-12-27T00:32:18.884951Z",
     "shell.execute_reply": "2024-12-27T00:32:18.883751Z"
    },
    "papermill": {
     "duration": 3036.917912,
     "end_time": "2024-12-27T00:32:18.887003",
     "exception": false,
     "start_time": "2024-12-26T23:41:41.969091",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "ERS = EarlyStopping(monitor='val_accuracy',min_delta=0,verbose=0,restore_best_weights = True,patience=10,mode='max')\n",
    "\n",
    "history = cnn_model.fit(train_x.reshape(train_x.shape[0], 28, 28, 1), train_y, epochs=100,validation_data=(test_x, test_y), callbacks=[ERS], verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "26b36f3b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-27T00:32:18.899801Z",
     "iopub.status.busy": "2024-12-27T00:32:18.899377Z",
     "iopub.status.idle": "2024-12-27T00:32:19.032527Z",
     "shell.execute_reply": "2024-12-27T00:32:19.031359Z"
    },
    "papermill": {
     "duration": 0.141849,
     "end_time": "2024-12-27T00:32:19.034503",
     "exception": false,
     "start_time": "2024-12-27T00:32:18.892654",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cnn_model.save('/kaggle/working/emnist_cnn_model.keras')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74ad2015",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-27T00:32:19.046867Z",
     "iopub.status.busy": "2024-12-27T00:32:19.046471Z",
     "iopub.status.idle": "2024-12-27T00:32:19.621037Z",
     "shell.execute_reply": "2024-12-27T00:32:19.619962Z"
    },
    "papermill": {
     "duration": 0.583364,
     "end_time": "2024-12-27T00:32:19.623542",
     "exception": false,
     "start_time": "2024-12-27T00:32:19.040178",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Plot training and validation accuracy\n",
    "plt.figure(figsize=(12, 6))\n",
    "\n",
    "# Plot training accuracy\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.plot(history.history['accuracy'], label='Train Accuracy')\n",
    "plt.plot(history.history['val_accuracy'], label='Validation Accuracy')\n",
    "plt.title('Train and Validation Accuracy')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "\n",
    "# Plot training and validation loss\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.plot(history.history['loss'], label='Train Loss')\n",
    "plt.plot(history.history['val_loss'], label='Validation Loss')\n",
    "plt.title('Train and Validation Loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "\n",
    "# Show the plot\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7724c756",
   "metadata": {},
   "source": [
    "## KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2920e2b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x_reshaped = train_x.reshape(train_x.shape[0], 28*28)\n",
    "train_y_int = np.argmax(train_y, axis=1)\n",
    "k_values = range(3, 12)\n",
    "cv_scores = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97856766",
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in k_values:\n",
    "    knn_model = KNeighborsClassifier(n_neighbors=k)\n",
    "    scores = cross_val_score(knn_model, train_x_reshaped, train_y_int, cv=5, scoring='accuracy')\n",
    "    cv_scores.append(scores.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1eec95db",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(k_values, cv_scores, marker='o')\n",
    "plt.title('KNN Cross-Validation Performance with Different k Values')\n",
    "plt.xlabel('k (Number of Neighbors)')\n",
    "plt.ylabel('Mean Accuracy (from Cross-Validation)')\n",
    "plt.xticks(k_values)\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "742f3c43",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_k = k_values[np.argmax(cv_scores)]\n",
    "best_knn_model = KNeighborsClassifier(n_neighbors=best_k, n_jobs=-1)\n",
    "best_knn_model.fit(train_x_reshaped, train_y_int)\n",
    "joblib.dump(best_knn_model, '/kaggle/working/emnist_knn_model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88dfbad6",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x_reshaped = test_x.reshape(test_x.shape[0], 28*28)\n",
    "test_y_int = np.argmax(test_y, axis=1)\n",
    "test_predictions = best_knn_model.predict(test_x_reshaped)\n",
    "test_accuracy = np.mean(test_predictions == test_y_int)\n",
    "\n",
    "print(f\"Test Accuracy with k={best_k}: {test_accuracy * 100:.2f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34b81f48",
   "metadata": {},
   "source": [
    "## SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1645f5c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "kernels = ['linear', 'rbf']\n",
    "C_values = [1, 0.1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "199ea5d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for kernel in kernels:\n",
    "    for C in C_values:\n",
    "        print(f\"Training SVM with kernel={kernel} and C={C}\")\n",
    "        \n",
    "        # Initialize the SVM model\n",
    "        model = SVC(kernel=kernel, C=C, random_state=42)\n",
    "        \n",
    "        # Train the model\n",
    "        model.fit(train_x_reshaped, train_y_int)\n",
    "        \n",
    "        # Save the model\n",
    "        model_filename = f'svm_model_{kernel}_C{C}.pkl'\n",
    "        joblib.dump(model, model_filename)\n",
    "        print(f\"Model saved as {model_filename}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1dc706e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = {\n",
    "    'C': [0.1, 1],\n",
    "    'kernel': ['linear', 'rbf']\n",
    "}\n",
    "grid = GridSearchCV(SVC(), param_grid, refit=True, verbose=2)\n",
    "grid.fit(train_x_reshaped, train_y_int)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37e77fb0",
   "metadata": {},
   "source": [
    "# Ensemble method (voting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "991ee90e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "ensemble_model = VotingClassifier(estimators=[\n",
    "    ('cnn', cnn_model),\n",
    "    ('knn', best_knn_model),\n",
    "    ('svm', grid.best_estimator_)\n",
    "], voting='hard')\n",
    "ensemble_model.fit(train_x_reshaped, train_y_int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db832134",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "test_predictions = cnn_model.predict(test_x)\n",
    "print(classification_report(test_y_int, test_predictions.argmax(axis=1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09580119",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "074fc6fd",
   "metadata": {},
   "source": [
    "## CNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78c71660",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Evaluate CNN model on test data\n",
    "cnn_test_predictions = cnn_model.predict(test_x)\n",
    "cnn_test_predictions_classes = cnn_test_predictions.argmax(axis=1)\n",
    "\n",
    "# Print classification report\n",
    "print(\"CNN Model Classification Report:\")\n",
    "print(classification_report(test_y_int, cnn_test_predictions_classes))\n",
    "\n",
    "# Plot confusion matrix\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.heatmap(confusion_matrix(test_y_int, cnn_test_predictions_classes), annot=True, fmt='d', cmap='Blues')\n",
    "plt.title('CNN Model Confusion Matrix')\n",
    "plt.xlabel('Predicted Labels')\n",
    "plt.ylabel('True Labels')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1f98126",
   "metadata": {},
   "source": [
    "## KNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19590c10",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate KNN model on test data\n",
    "knn_test_predictions = best_knn_model.predict(test_x_reshaped)\n",
    "\n",
    "# Print classification report\n",
    "print(f\"KNN Model Classification Report (k={best_k}):\")\n",
    "print(classification_report(test_y_int, knn_test_predictions))\n",
    "\n",
    "# Plot confusion matrix\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.heatmap(confusion_matrix(test_y_int, knn_test_predictions), annot=True, fmt='d', cmap='Blues')\n",
    "plt.title(f'KNN Model Confusion Matrix (k={best_k})')\n",
    "plt.xlabel('Predicted Labels')\n",
    "plt.ylabel('True Labels')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20bc473d",
   "metadata": {},
   "source": [
    "## SVM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b8f3867",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the best SVM model (assuming it's saved as 'svm_model_rbf_C1.pkl')\n",
    "best_svm_model = grid.best_estimator_\n",
    "\n",
    "# Evaluate SVM model on test data\n",
    "svm_test_predictions = best_svm_model.predict(test_x_reshaped)\n",
    "\n",
    "# Print classification report\n",
    "print(\"SVM Model Classification Report (kernel=rbf, C=1):\")\n",
    "print(classification_report(test_y_int, svm_test_predictions))\n",
    "\n",
    "# Plot confusion matrix\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.heatmap(confusion_matrix(test_y_int, svm_test_predictions), annot=True, fmt='d', cmap='Blues')\n",
    "plt.title('SVM Model Confusion Matrix (kernel=rbf, C=1)')\n",
    "plt.xlabel('Predicted Labels')\n",
    "plt.ylabel('True Labels')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00d79390",
   "metadata": {},
   "source": [
    "## Ensemble Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e48dac73",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate Ensemble model on test data\n",
    "ensemble_test_predictions = ensemble_model.predict(test_x_reshaped)\n",
    "\n",
    "# Print classification report\n",
    "print(\"Ensemble Model Classification Report:\")\n",
    "print(classification_report(test_y_int, ensemble_test_predictions))\n",
    "\n",
    "# Plot confusion matrix\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.heatmap(confusion_matrix(test_y_int, ensemble_test_predictions), annot=True, fmt='d', cmap='Blues')\n",
    "plt.title('Ensemble Model Confusion Matrix')\n",
    "plt.xlabel('Predicted Labels')\n",
    "plt.ylabel('True Labels')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "sourceId": 214803128,
     "sourceType": "kernelVersion"
    }
   ],
   "dockerImageVersionId": 30823,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 3060.059461,
   "end_time": "2024-12-27T00:32:21.257952",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-12-26T23:41:21.198491",
   "version": "2.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}