Project 13 - custom activation func using Lambda.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a19df0af",
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "  # %tensorflow_version only exists in Colab.\n",
    "  %tensorflow_version 2.x\n",
    "except Exception:\n",
    "  pass\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d151b560",
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = tf.keras.datasets.mnist\n",
    "\n",
    "(x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
    "x_train, x_test = x_train / 255.0, x_test / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b6b38473",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 28, 28), (60000,))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape , y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32f22b05",
   "metadata": {},
   "source": [
    "## Build the Model\n",
    "\n",
    "Here, we'll use a Lambda layer to define a custom layer in our network. We're using a lambda function to get the absolute value of the layer input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8391a961",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential([\n",
    "  tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "  tf.keras.layers.Dense(128),\n",
    "  tf.keras.layers.Lambda(lambda x: tf.abs(x)), \n",
    "  tf.keras.layers.Dense(10, activation='softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "60fd2d81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1875/1875 [==============================] - 9s 4ms/step - loss: 0.2305 - accuracy: 0.9339\n",
      "Epoch 2/5\n",
      "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0922 - accuracy: 0.9720\n",
      "Epoch 3/5\n",
      "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0645 - accuracy: 0.9799\n",
      "Epoch 4/5\n",
      "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0488 - accuracy: 0.9844\n",
      "Epoch 5/5\n",
      "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0385 - accuracy: 0.9877\n",
      "313/313 [==============================] - 2s 5ms/step - loss: 0.0909 - accuracy: 0.9739\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.09094060212373734, 0.9739000201225281]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train, epochs=5)\n",
    "model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86cf6234",
   "metadata": {},
   "source": [
    "Another way to use the Lambda layer is to pass in a function defined outside the model. The code below shows how a custom ReLU function is used as a custom layer in the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "da49892c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1875/1875 [==============================] - 7s 4ms/step - loss: 0.4265 - accuracy: 0.8798\n",
      "Epoch 2/5\n",
      "1875/1875 [==============================] - 7s 4ms/step - loss: 0.1228 - accuracy: 0.9638\n",
      "Epoch 3/5\n",
      "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0767 - accuracy: 0.9773\n",
      "Epoch 4/5\n",
      "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0553 - accuracy: 0.9831\n",
      "Epoch 5/5\n",
      "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0426 - accuracy: 0.9872\n",
      "313/313 [==============================] - 1s 4ms/step - loss: 0.0790 - accuracy: 0.9759: 1s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.07900049537420273, 0.9758999943733215]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def my_relu(x):\n",
    "    return K.maximum(-0.1, x)\n",
    "\n",
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "    tf.keras.layers.Dense(128),\n",
    "    tf.keras.layers.Lambda(my_relu), \n",
    "    tf.keras.layers.Dense(10, activation='softmax')\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train, epochs=5)\n",
    "model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d86cedea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1875/1875 [==============================] - 12s 6ms/step - loss: 0.2669 - accuracy: 0.9233\n",
      "Epoch 2/5\n",
      "1875/1875 [==============================] - 12s 6ms/step - loss: 0.1195 - accuracy: 0.9641\n",
      "Epoch 3/5\n",
      "1875/1875 [==============================] - 12s 6ms/step - loss: 0.0799 - accuracy: 0.9761\n",
      "Epoch 4/5\n",
      "1875/1875 [==============================] - 12s 6ms/step - loss: 0.0584 - accuracy: 0.9819\n",
      "Epoch 5/5\n",
      "1875/1875 [==============================] - 12s 6ms/step - loss: 0.0457 - accuracy: 0.9855\n",
      "313/313 [==============================] - 2s 5ms/step - loss: 0.0781 - accuracy: 0.9765\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.0780668705701828, 0.9764999747276306]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def my_relu(x):\n",
    "    return K.maximum(-0.5, x)\n",
    "\n",
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "    tf.keras.layers.Dense(128),\n",
    "    tf.keras.layers.Lambda(my_relu), \n",
    "    tf.keras.layers.Dense(10, activation='softmax')\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train, epochs=5)\n",
    "model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d83dfd4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}