Signature-Verification-Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "8GHc5YhGHSTN",
    "outputId": "e9917f01-a7e9-40a7-fe1e-62ce82e5acca"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8Ncb5xPXHcbg"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "loc = 'drive/My Drive/Signature_Verification'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3z6kkZMyHlhD"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import random\n",
    "from scipy import stats\n",
    "\n",
    "from data_processing import *\n",
    "\n",
    "from tensorflow.keras.layers import BatchNormalization\n",
    "from tensorflow.keras.layers import MaxPooling2D\n",
    "from tensorflow.keras.layers import ZeroPadding2D\n",
    "from tensorflow.keras.layers import Dropout\n",
    "\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.optimizers import Adam, RMSprop\n",
    "from tensorflow.keras.layers import Conv2D, ZeroPadding2D, Activation, Input, concatenate, Dropout\n",
    "from tensorflow.keras.layers import BatchNormalization, MaxPool2D,Concatenate,Lambda,Flatten,Dense\n",
    "from tensorflow.keras.initializers import glorot_uniform\n",
    "\n",
    "from tensorflow.keras.models import Model\n",
    "\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 124
    },
    "colab_type": "code",
    "id": "4MZ2jS_ftXRw",
    "outputId": "60ccc356-8636-4aeb-f277-5eef7768a7c9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.config',\n",
       " '__pycache__',\n",
       " 'BHSig260.zip',\n",
       " 'data_processing.py',\n",
       " 'drive',\n",
       " 'sample_data']"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nubdXr6olh8o"
   },
   "source": [
    "# Dataset Generation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qN1Ln6hWlEam"
   },
   "source": [
    "## Importing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mC9CV3F8IDxN"
   },
   "outputs": [],
   "source": [
    "imgsize = (150,150)\n",
    "size= 150\n",
    "path_dataset1 = os.path.join(loc,'Signature_Set1')\n",
    "path_dataset2 = os.path.join(loc,'Signature_Set2')\n",
    "path_dataset3 = 'Signature_Set3'\n",
    "images_dictionary = {} # This dictionary stores all the images, to avoid too much memory consumption"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZqsmculNnGFK"
   },
   "outputs": [],
   "source": [
    "fname = 'BHSig260.zip'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "L-YBPxlhkha8"
   },
   "source": [
    "### Dataset from Set-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "id": "m26B8JUxISe9",
    "outputId": "0f8214f5-f38e-43b6-9008-692909342067"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Dataset-1 Data and Saving inside the Dictionary..\n",
      "Getting Genuine Images..\n",
      "25% Completed..\n",
      "50% Completed..\n",
      "100% Completed\n",
      "Getting Forged Images..\n",
      "25% Completed..\n",
      "50% Completed..\n",
      "100% Completed\n",
      "600 600 600\n"
     ]
    }
   ],
   "source": [
    "real_img,forged_img = getImages(path_dataset1,imgsize,images_dictionary)\n",
    "real_img = np.sort(real_img)\n",
    "forged_img = np.sort(forged_img)\n",
    "set1_X1,set1_X2,set1_Y = makePairs(real_img,forged_img,5)\n",
    "print(len(set1_X1),len(set1_X2),len(set1_Y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HI8p60FdMsIK"
   },
   "source": [
    "### Dataset from Set-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 212
    },
    "colab_type": "code",
    "id": "lnSg-TlkMuJM",
    "outputId": "95608738-2bc5-4fa0-8115-baeedbe2c60a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Dataset2 Data..\n",
      "25% Completed..\n",
      "50% Completed..\n",
      "Couldn't import  Thumbs.db in Location: drive/My Drive/Signature_Verification/Signature_Set2/full_org\n",
      "100% Completed\n",
      "25% Completed..\n",
      "50% Completed..\n",
      "Couldn't import  Thumbs.db in Location: drive/My Drive/Signature_Verification/Signature_Set2/full_forg\n",
      "100% Completed\n",
      "Data Import Complete!\n",
      "30360 30360 30360\n"
     ]
    }
   ],
   "source": [
    "real_img,forged_img = getImages2(path_dataset2,imgsize,images_dictionary)\n",
    "real_img = np.sort(real_img)\n",
    "forged_img = np.sort(forged_img)\n",
    "set2_X1,set2_X2,set2_Y = makePairs(real_img,forged_img,24)\n",
    "print(len(set2_X1),len(set2_X2),len(set2_Y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aunbqAGHko1_"
   },
   "source": [
    "### Dataset from Set-3 (BHSig260)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "id": "Z7yYGlrTnGI0",
    "outputId": "e44453a4-a876-486e-e6ad-af7444271c28"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Genuine Data..\n",
      "25% Complete\n",
      "50% Complete\n",
      "75% Complete\n",
      "100% Complete\n",
      "Getting Forged Data..\n",
      "25% Complete\n",
      "50% Complete\n",
      "75% Complete\n",
      "100% Complete\n"
     ]
    }
   ],
   "source": [
    "real,forged = getHindi(fname,imgsize,images_dictionary)\n",
    "X1H,X2H,yH  = makeHindiPairs(real,forged)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NpY_ncIekzjP"
   },
   "source": [
    "Joining the Set2 and Set3 Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "KxWcz-CUM-52",
    "outputId": "b9b94e2f-daa6-43ee-c39d-b420e21f6eca"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "189720 189720 189720\n"
     ]
    }
   ],
   "source": [
    "X1 = np.concatenate([X1H,set2_X1])\n",
    "X2 = np.concatenate([X2H,set2_X2])\n",
    "Y  = np.concatenate([yH,set2_Y])\n",
    "print(len(X1),len(X2),len(Y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2HgX0xvuH8Ts"
   },
   "source": [
    "#### Shuffling Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xjpQakLlH7gS"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "random.seed=35\n",
    "mapIndexPosition = list(zip(X1,X2,Y))\n",
    "random.shuffle(mapIndexPosition)\n",
    "X1,X2,Y = zip(*mapIndexPosition)\n",
    "Y = tf.cast(Y, tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "owCJbO7uOXdj"
   },
   "outputs": [],
   "source": [
    "X1_train = X1[0:173720]\n",
    "X2_train = X2[0:173720]\n",
    "Y_train  = Y[0:173720]\n",
    "X1_val   = X1[173720:]\n",
    "X2_val   = X2[173720:]\n",
    "Y_val    = Y[173720:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "c1p4VXxwxKro"
   },
   "source": [
    "## Generating Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gQSCp872IiBP"
   },
   "outputs": [],
   "source": [
    "def create_batch(x1,x2,startpoint,batch_size):\n",
    "    fir=np.zeros((batch_size,imgsize[0],imgsize[0],1))\n",
    "    sec=np.zeros((batch_size,imgsize[0],imgsize[0],1))\n",
    "    counter=0\n",
    "    for i in range(startpoint,startpoint+batch_size):\n",
    "        fir[counter]=images_dictionary[x1[i]]\n",
    "        sec[counter]=images_dictionary[x2[i]]\n",
    "        counter+=1\n",
    "    return fir,sec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NlZS9pRq4Pn7"
   },
   "outputs": [],
   "source": [
    "# Function takes in input of array of names, and returns the image list with expanded dimenstions\n",
    "def returnImageList(name_lis):\n",
    "    length = len(name_lis)\n",
    "    imgs = np.zeros((length,imgsize[0],imgsize[0],1))\n",
    "    for i in range(0,length):\n",
    "        imgs[i] = images_dictionary[name_lis[i]]\n",
    "    return imgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7R-1-IZO4T5e"
   },
   "outputs": [],
   "source": [
    "def data_generator(batch_size,x1,x2,y):\n",
    "    counter=0\n",
    "    while True:\n",
    "        if counter>=len(y): counter=0\n",
    "        a,b = create_batch(x1,x2,counter,batch_size)\n",
    "        y1 = y[counter:counter+batch_size]\n",
    "        counter+=batch_size\n",
    "        yield [a,b],y1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "On9SHcf6llhj"
   },
   "source": [
    "# Preparing the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WyZZA61LyyCw"
   },
   "source": [
    "### Embedding Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 959
    },
    "colab_type": "code",
    "id": "OxxyWMAYybKn",
    "outputId": "591c4bc8-3f1f-4647-c50c-8f5c54cb79ff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"embedding_model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Layer1 (Conv2D)              (None, 148, 148, 64)      640       \n",
      "_________________________________________________________________\n",
      "leaky_re_lu (LeakyReLU)      (None, 148, 148, 64)      0         \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 148, 148, 64)      592       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 74, 74, 64)        0         \n",
      "_________________________________________________________________\n",
      "Layer2 (Conv2D)              (None, 72, 72, 128)       73856     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)    (None, 72, 72, 128)       0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 72, 72, 128)       288       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 36, 36, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 36, 36, 128)       0         \n",
      "_________________________________________________________________\n",
      "Layer3 (Conv2D)              (None, 34, 34, 256)       295168    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)    (None, 34, 34, 256)       0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 34, 34, 256)       136       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "Layer4 (Conv2D)              (None, 6, 6, 512)         1180160   \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)    (None, 6, 6, 512)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 6, 6, 512)         24        \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 1, 1, 512)         0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 1, 1, 512)         0         \n",
      "_________________________________________________________________\n",
      "Layer5 (Conv2D)              (None, 1, 1, 512)         262656    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_4 (LeakyReLU)    (None, 1, 1, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 512)               0         \n",
      "=================================================================\n",
      "Total params: 1,813,520\n",
      "Trainable params: 1,813,000\n",
      "Non-trainable params: 520\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "emb_dim = 64\n",
    "\n",
    "embedding_model = tf.keras.models.Sequential([\n",
    "  tf.keras.layers.Conv2D(64, (3,3), input_shape=(size,size,1),name='Layer1'),\n",
    "  tf.keras.layers.LeakyReLU(),\n",
    "  tf.keras.layers.BatchNormalization(epsilon=1e-06, axis=1, momentum=0.9),\n",
    "  tf.keras.layers.MaxPooling2D(2, 2),\n",
    "  #tf.keras.layers.Dropout(0.2),\n",
    "    \n",
    "  tf.keras.layers.Conv2D(128, (3,3),name='Layer2'),\n",
    "  tf.keras.layers.LeakyReLU(),\n",
    "  tf.keras.layers.BatchNormalization(epsilon=1e-06, axis=1, momentum=0.9),\n",
    "  tf.keras.layers.MaxPooling2D(2, 2),\n",
    "  tf.keras.layers.Dropout(0.1),\n",
    "    \n",
    "  tf.keras.layers.Conv2D(256, (3,3),name='Layer3'),\n",
    "  tf.keras.layers.LeakyReLU(),\n",
    "  tf.keras.layers.BatchNormalization(epsilon=1e-06, axis=1, momentum=0.9),\n",
    "  tf.keras.layers.MaxPooling2D(4, 4),\n",
    "  tf.keras.layers.Dropout(0.2),\n",
    "\n",
    "  tf.keras.layers.Conv2D(512, (3,3),name='Layer4'),\n",
    "  tf.keras.layers.LeakyReLU(),\n",
    "  tf.keras.layers.BatchNormalization(epsilon=1e-06, axis=1, momentum=0.9),\n",
    "  tf.keras.layers.MaxPooling2D(4, 4),\n",
    "  tf.keras.layers.Dropout(0.1),\n",
    "\n",
    "  tf.keras.layers.Conv2D(512, (1,1),name='Layer5'),\n",
    "  tf.keras.layers.LeakyReLU(),\n",
    "  tf.keras.layers.Flatten(),\n",
    "],name='embedding_model')\n",
    "print(embedding_model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "z5PjAnSo0HOY"
   },
   "source": [
    "### Siamese-Type Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 461
    },
    "colab_type": "code",
    "id": "T8AYxfwzz6Gp",
    "outputId": "2be17aad-2247-4d38-973c-4f627f111f11"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input1 (InputLayer)             [(None, 150, 150, 1) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input2 (InputLayer)             [(None, 150, 150, 1) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_model (Sequential)    (None, 512)          1813520     input1[0][0]                     \n",
      "                                                                 input2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 1024)         0           embedding_model[1][0]            \n",
      "                                                                 embedding_model[2][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 512)          524800      concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 64)           32832       dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "Output (Dense)                  (None, 1)            65          dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 2,371,217\n",
      "Trainable params: 2,370,697\n",
      "Non-trainable params: 520\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_a = tf.keras.layers.Input(shape=(size,size,1),name='input1')\n",
    "input_b = tf.keras.layers.Input(shape=(size,size,1),name='input2')\n",
    "\n",
    "em_one = embedding_model(input_a)\n",
    "em_two = embedding_model(input_b)\n",
    "\n",
    "out = tf.keras.layers.concatenate([em_one,em_two],axis=1)\n",
    "out = tf.keras.layers.Dense(512,activation='relu',kernel_regularizer=tf.keras.regularizers.l2(l=0.1))(out)\n",
    "out = tf.keras.layers.Dense(64,activation='relu')(out)\n",
    "\n",
    "out = tf.keras.layers.Dense(1,activation='sigmoid',name='Output')(out)\n",
    "\n",
    "model = tf.keras.models.Model([input_a, input_b],out)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MOt6_79Pmv34"
   },
   "source": [
    "### Learning Rate Scheduling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7KYU-tH_mvHx"
   },
   "outputs": [],
   "source": [
    "initial_learning_rate = 0.0008\n",
    "lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n",
    "    initial_learning_rate=initial_learning_rate,\n",
    "    decay_steps=5, \n",
    "    decay_rate=0.96,\n",
    "    staircase=True\n",
    ") #lr = lr * (decay_rate ^ decay_steps)\n",
    "optimizer= tf.keras.optimizers.RMSprop(learning_rate=lr_schedule)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KzH2UaZj16jm"
   },
   "outputs": [],
   "source": [
    "batch_size = 8\n",
    "epoch = 4\n",
    "rms = tf.keras.optimizers.RMSprop(lr=1e-4, rho=0.9, epsilon=1e-08)\n",
    "model.compile(loss='binary_crossentropy', optimizer=optimizer,metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 159
    },
    "colab_type": "code",
    "id": "FRGEJ5R95K5h",
    "outputId": "35dac180-82e8-4c04-84da-d2070f299473"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/4\n",
      "21715/21715 [==============================] - 2165s 100ms/step - loss: 0.6043 - accuracy: 0.7661 - val_loss: 0.5550 - val_accuracy: 0.7197\n",
      "Epoch 2/4\n",
      "21715/21715 [==============================] - 2170s 100ms/step - loss: 0.5252 - accuracy: 0.7683 - val_loss: 0.5549 - val_accuracy: 0.7199\n",
      "Epoch 3/4\n",
      "21715/21715 [==============================] - 2188s 101ms/step - loss: 0.5254 - accuracy: 0.7677 - val_loss: 0.5549 - val_accuracy: 0.7199\n",
      "Epoch 4/4\n",
      "21715/21715 [==============================] - 2192s 101ms/step - loss: 0.5253 - accuracy: 0.7680 - val_loss: 0.5548 - val_accuracy: 0.7200\n"
     ]
    }
   ],
   "source": [
    "_ =model.fit(\n",
    "    data_generator(batch_size,X1_train,X2_train,Y_train),\n",
    "    epochs=epoch,\n",
    "    steps_per_epoch=len(Y_train) // batch_size,\n",
    "    validation_data=data_generator(batch_size,X1_val,X2_val,Y_val),\n",
    "    validation_steps = len(Y_val) // batch_size,\n",
    "    verbose=1\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rWZ5oozBfHYd"
   },
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5uUXVVJ1fMtY"
   },
   "source": [
    "### On Training Set\n",
    "*(Accuracy suggests that the model has avoided overfitting along with learning)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "NzMOS5ucfL36",
    "outputId": "081facdb-c039-418a-dceb-04b0fc567d4a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2000/2000 [==============================] - 75s 37ms/step - loss: 0.5554 - accuracy: 0.7153\n"
     ]
    }
   ],
   "source": [
    "train_set_accuracy = model.evaluate(data_generator(batch_size,X1_train,X2_train,Y_train),batch_size=batch_size,steps=len(Y_val) // batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "Um-MTnWbPVNd",
    "outputId": "c030a879-3a6f-4829-8c51-d509b53aa4ea"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set Accuracy is :  71.53124809265137\n"
     ]
    }
   ],
   "source": [
    "print(\"Training Set Accuracy is : \", train_set_accuracy[1]*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pCirBOHRf2Bz"
   },
   "source": [
    "### On Validation Set\n",
    "*(Accuracy suggests how well the model has actually learnt)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "IEEGemvSX3R3",
    "outputId": "ed726ac1-5ed1-47c8-c50c-d215705be209"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2000/2000 [==============================] - 75s 37ms/step - loss: 0.5548 - accuracy: 0.7200\n"
     ]
    }
   ],
   "source": [
    "validation_set_accuracy=model.evaluate(data_generator(batch_size,X1_val,X2_val,Y_val),batch_size=batch_size,steps=len(Y_val) // batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "qOZIQKhdYE0V",
    "outputId": "a4b2131b-afd8-41ca-d41e-6eabf63fa17f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set Accuracy =  72.00000286102295\n"
     ]
    }
   ],
   "source": [
    "print(\"Validation Set Accuracy = \",validation_set_accuracy[1]*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FO2N2umggHXU"
   },
   "source": [
    "### On Dataset which is totally new for the model\n",
    "*(Expected Accuracy is very low, just to get an idea on how well the model has learnt)* <br> <br>\n",
    "**NOTE: This dataset is not considered as test set, as this dataset(Containing full english signatures) is not similar to the training set(Mostly having Hindi Language Signatures).** <br><br>\n",
    "This Accuracy is calculated based on threshold. Since the result are the probability (by sigmoid activation function), threshold using the **mean** of all the values gave comparatively better accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "45LqJSKAbM1r"
   },
   "outputs": [],
   "source": [
    "test_preds = model.predict([returnImageList(set1_X1),returnImageList(set1_X2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sU03saItgana"
   },
   "outputs": [],
   "source": [
    "def calculateAccuracy(Y,predictions,threshold): # Calculates Accuracy on a given threshold value\n",
    "    test_preds_final = []\n",
    "    for i in predictions:\n",
    "        if i[0]<0.35: test_preds_final.append(0)\n",
    "        else: test_preds_final.append(1)\n",
    "\n",
    "    counter=0\n",
    "    for i in range(0,len(set1_Y)):\n",
    "        if set1_Y[i] == test_preds_final[i]:\n",
    "            counter+=1\n",
    "    return counter*100/len(test_preds_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "jlWR1k6hckcY",
    "outputId": "8005b3ab-4907-47a2-9807-deeaa8370dd0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.49748087\n"
     ]
    }
   ],
   "source": [
    "mean = np.mean(test_preds)\n",
    "print(mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "3PbLQJM_crRX",
    "outputId": "e60c6822-c823-4128-8ddd-3bc368f7a1f9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy gained by the totally new dataset is :  57.5\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy gained by the totally new dataset is : \",calculateAccuracy(set1_Y,test_preds,np.mean(test_preds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "o7T2a3s-g-X2"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "SignVer Good Performance.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}