part1_feature_extraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "colab": {
      "name": "part1-feature-extraction.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5cdcbac022944149998b5876efff5ff2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a899a5d3e5c6455aa506a70e7ccbc552",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_78cefd8f510c49179442297df3510d2b",
              "IPY_MODEL_dc5cef09bb77402aabd034307fefb20f"
            ]
          }
        },
        "a899a5d3e5c6455aa506a70e7ccbc552": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "78cefd8f510c49179442297df3510d2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d3f3ec80baa0495d96ae2bec2bda64cf",
            "_dom_classes": [],
            "description": "  3%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 8677,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 226,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1f14c5f699334d42a9241b1cf4249489"
          }
        },
        "dc5cef09bb77402aabd034307fefb20f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_76f4f5853a8a4e94a5fe9e671163c1ed",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 226/8677 [02:01&lt;1:17:17,  1.82it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ce19348d4e184221886a72997d084f4c"
          }
        },
        "d3f3ec80baa0495d96ae2bec2bda64cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1f14c5f699334d42a9241b1cf4249489": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "76f4f5853a8a4e94a5fe9e671163c1ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ce19348d4e184221886a72997d084f4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/xiaoya27/ANN-CBIR/blob/main/part1_feature_extraction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y7a8jN37Vqhu"
      },
      "source": [
        "# Feature Extraction\n",
        "Extract features from pretrained models like VGG-16, VGG-19, ResNet-50, InceptionV3 and MobileNet and benchmark them using the Caltech101 dataset.\n",
        "\n",
        "## Dataset:\n",
        "\n",
        "\n",
        "```\n",
        " https://drive.google.com/uc?id=137RyRjvTBkBiIfeYBNZBtViDHQ6_Ewsp=\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jl7y3tPaVqhu"
      },
      "source": [
        "!mkdir -p /content/drive/MyDrive/pipline-data/datasets\n",
        "!pip install gdown\n",
        "/content/drive/MyDrive/pipline-data\n",
        "!gdown https://drive.google.com/uc?id=137RyRjvTBkBiIfeYBNZBtViDHQ6_Ewsp --output /content/drive/MyDrive/pipline-data/datasets/caltech101.tar.gz\n",
        "!tar -xvzf /content/drive/MyDrive/pipline-data/datasets/caltech101.tar.gz --directory /content/drive/MyDrive/pipline-data/datasets\n",
        "!mv /content/drive/MyDrive/pipline-data/datasets/101_ObjectCategories /content/drive/MyDrive/pipline-data/datasets/caltech101\n",
        "!rm -rf /content/drive/MyDrive/pipline-data/datasets/caltech101/BACKGROUND_Google"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0Mbn4drF_GoP",
        "outputId": "3cbdda21-ddb7-4923-d66f-f13f55aa4d64"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TQIPEFy7__3n",
        "outputId": "1602f28a-10f3-4323-dc1e-511b36d7021b"
      },
      "source": [
        "%cd /content/drive/MyDrive/pipline-data\n",
        "!pwd"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/pipline-data\n",
            "/content/drive/MyDrive/pipline-data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": false,
        "id": "UhvdKsG_Vqhu"
      },
      "source": [
        "import numpy as np\n",
        "from numpy.linalg import norm\n",
        "import pickle\n",
        "from tqdm.notebook import tqdm\n",
        "from tqdm import tqdm_notebook\n",
        "import os\n",
        "import random\n",
        "import time\n",
        "import math\n",
        "import tensorflow\n",
        "from tensorflow.keras.preprocessing import image\n",
        "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
        "from tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input\n",
        "from tensorflow.keras.applications.vgg16 import VGG16\n",
        "from tensorflow.keras.applications.vgg19 import VGG19\n",
        "from tensorflow.keras.applications.mobilenet import MobileNet\n",
        "from tensorflow.keras.applications.inception_v3 import InceptionV3\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, GlobalAveragePooling2D\n"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HYOHdmqOVqhu"
      },
      "source": [
        " # define a helper function that allows us to choose any pretrained model with all the necessary details for our experiments.\n",
        " \n",
        " def model_picker(name):\n",
        "    if (name == 'vgg16'):\n",
        "        model = VGG16(weights='imagenet',\n",
        "                      include_top=False,\n",
        "                      input_shape=(224, 224, 3),\n",
        "                      pooling='max')\n",
        "    elif (name == 'vgg19'):\n",
        "        model = VGG19(weights='imagenet',\n",
        "                      include_top=False,\n",
        "                      input_shape=(224, 224, 3),\n",
        "                      pooling='max')\n",
        "    elif (name == 'mobilenet'):\n",
        "        model = MobileNet(weights='imagenet',\n",
        "                          include_top=False,\n",
        "                          input_shape=(224, 224, 3),\n",
        "                          pooling='max',\n",
        "                          depth_multiplier=1,\n",
        "                          alpha=1)\n",
        "    elif (name == 'inception'):\n",
        "        model = InceptionV3(weights='imagenet',\n",
        "                            include_top=False,\n",
        "                            input_shape=(224, 224, 3),\n",
        "                            pooling='max')\n",
        "    elif (name == 'resnet'):\n",
        "        model = ResNet50(weights='imagenet',\n",
        "                         include_top=False,\n",
        "                         input_shape=(224, 224, 3),\n",
        "                        pooling='max')\n",
        "    elif (name == 'xception'):\n",
        "        model = Xception(weights='imagenet',\n",
        "                         include_top=False,\n",
        "                         input_shape=(224, 224, 3),\n",
        "                         pooling='max')\n",
        "    else:\n",
        "        print(\"Specified model not available\")\n",
        "    return model"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9MydWqJJVqhu"
      },
      "source": [
        "Now, let's put our function to use."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "39_9Sn9-Vqhu"
      },
      "source": [
        "model_architecture = 'mobilenet'\n",
        "model = model_picker(model_architecture)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hyvlmkO_Vqhu"
      },
      "source": [
        " define a function to extract image features given an image and a model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5tHvcKs-Vqhu"
      },
      "source": [
        "def extract_features(img_path, model):\n",
        "    input_shape = (224, 224, 3)\n",
        "    img = image.load_img(img_path,\n",
        "                         target_size=(input_shape[0], input_shape[1]))\n",
        "    img_array = image.img_to_array(img)\n",
        "    expanded_img_array = np.expand_dims(img_array, axis=0)\n",
        "    preprocessed_img = preprocess_input(expanded_img_array)\n",
        "    features = model.predict(preprocessed_img)\n",
        "    flattened_features = features.flatten()\n",
        "    normalized_features = flattened_features / norm(flattened_features)\n",
        "    return normalized_features"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qnql_Vt9Vqhu",
        "outputId": "e0a8b242-a400-46c6-fbc6-975ab5bd8d30"
      },
      "source": [
        "# check the the feature length the model generates.\n",
        "features = extract_features('datasets/caltech101/ant/image_0002.jpg', model)\n",
        "print(len(features))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1024\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sJ_Zv-H4Vqhu"
      },
      "source": [
        "Now, we will see how much time it takes to extract features of one image."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jhkt2R9KVqhu",
        "outputId": "89db5bed-bb6b-49a7-eb20-505f07bb1300"
      },
      "source": [
        "%timeit features = extract_features('datasets/caltech101/ant/image_0001.jpg', model)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The slowest run took 17.27 times longer than the fastest. This could mean that an intermediate result is being cached.\n",
            "1 loop, best of 3: 29.4 ms per loop\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZKNVE51RVqhu"
      },
      "source": [
        "The time taken to extract features is dependent on a few factors such as image size, computing power etc. A better benchmark would be running the network over an entire dataset. A simple change to the existing code will allow this.\n",
        "\n",
        "Let's make a handy function to recursively get all the image files under a root directory."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JCdIEgNnVqhu"
      },
      "source": [
        "extensions = ['.jpg', '.JPG', '.jpeg', '.JPEG', '.png', '.PNG']\n",
        "\n",
        "def get_file_list(root_dir):\n",
        "    file_list = []\n",
        "    for root, directories, filenames in os.walk(root_dir):\n",
        "        for filename in filenames:\n",
        "            if any(ext in filename for ext in extensions):\n",
        "                file_list.append(os.path.join(root, filename))\n",
        "    return file_list"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Edvs43iVqhu"
      },
      "source": [
        "Now, let's run the extraction over the entire dataset and time it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WMt1od7aD-fT"
      },
      "source": [
        "# path to the your datasets\n",
        "root_dir = 'datasets/caltech101'"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 436,
          "referenced_widgets": [
            "5cdcbac022944149998b5876efff5ff2",
            "a899a5d3e5c6455aa506a70e7ccbc552",
            "78cefd8f510c49179442297df3510d2b",
            "dc5cef09bb77402aabd034307fefb20f",
            "d3f3ec80baa0495d96ae2bec2bda64cf",
            "1f14c5f699334d42a9241b1cf4249489",
            "76f4f5853a8a4e94a5fe9e671163c1ed",
            "ce19348d4e184221886a72997d084f4c"
          ]
        },
        "id": "Izg-VRpNVqhu",
        "outputId": "ed05ca8c-0c2c-4693-f9c6-9a65190be403"
      },
      "source": [
        "filenames = sorted(get_file_list(root_dir))\n",
        "\n",
        "feature_list = []\n",
        "for i in tqdm_notebook(range(len(filenames))):\n",
        "    feature_list.append(extract_features(filenames[i], model))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5cdcbac022944149998b5876efff5ff2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=8677.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-185170a27d8f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mfeature_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm_notebook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilenames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mfeature_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextract_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilenames\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-6-3e7e21823682>\u001b[0m in \u001b[0;36mextract_features\u001b[0;34m(img_path, model)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0minput_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m224\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m224\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     img = image.load_img(img_path,\n\u001b[0;32m----> 4\u001b[0;31m                          target_size=(input_shape[0], input_shape[1]))\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mimg_array\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimg_to_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mexpanded_img_array\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg_array\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/preprocessing/image.py\u001b[0m in \u001b[0;36mload_img\u001b[0;34m(path, grayscale, color_mode, target_size, interpolation)\u001b[0m\n\u001b[1;32m    299\u001b[0m   \"\"\"\n\u001b[1;32m    300\u001b[0m   return image.load_img(path, grayscale=grayscale, color_mode=color_mode,\n\u001b[0;32m--> 301\u001b[0;31m                         target_size=target_size, interpolation=interpolation)\n\u001b[0m\u001b[1;32m    302\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/utils.py\u001b[0m in \u001b[0;36mload_img\u001b[0;34m(path, grayscale, color_mode, target_size, interpolation)\u001b[0m\n\u001b[1;32m    112\u001b[0m                           'The use of `load_img` requires PIL.')\n\u001b[1;32m    113\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m         \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpil_image\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mBytesIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcolor_mode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'grayscale'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m             \u001b[0;31m# if image is not already an 8-bit, 16-bit or 32-bit grayscale image\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uJDwgJdmC_Sq",
        "outputId": "d3e6d7a3-4065-4789-eeb8-000516db054d"
      },
      "source": [
        "import tensorflow as tf\n",
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tkoe87ckVqhu"
      },
      "source": [
        "Now let's try the same with the Keras Image Generator functions."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "46tDhSLiVqhu",
        "outputId": "16a89cd1-6aa8-4691-8720-d0198d79edb0"
      },
      "source": [
        "#with tf.device('/device:GPU:0'):\n",
        "batch_size = 64\n",
        "datagen = tf.keras.preprocessing.image.ImageDataGenerator(preprocessing_function=preprocess_input)  \n",
        "generator = datagen.flow_from_directory(root_dir,\n",
        "                                        target_size=(224, 224),\n",
        "                                        batch_size=batch_size,\n",
        "                                        class_mode=None,\n",
        "                                        shuffle=False)\n",
        "\n",
        "num_images = len(generator.filenames)\n",
        "num_epochs = int(math.ceil(num_images / batch_size))\n",
        "\n",
        "start_time = time.time()\n",
        "feature_list = []\n",
        "feature_list = model.predict(generator, num_epochs)\n",
        "end_time = time.time()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 8677 images belonging to 101 classes.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jLLJdnQSVqhu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8d22c88f-2ff0-43b3-9361-4549cfa0e025"
      },
      "source": [
        "for i, features in enumerate(feature_list):\n",
        "    feature_list[i] = features / norm(features)\n",
        "\n",
        "feature_list = feature_list.reshape(num_images, -1)\n",
        "\n",
        "print(\"Num images   = \", len(generator.classes))\n",
        "print(\"Shape of feature_list = \", feature_list.shape)\n",
        "print(\"Time taken in sec = \", end_time - start_time)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Num images   =  8677\n",
            "Shape of feature_list =  (8677, 1024)\n",
            "Time taken in sec =  3844.544595479965\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8be9OWPkVqhu"
      },
      "source": [
        "### GPU Utilization's effect on time taken by varying batch size \n",
        "\n",
        "\n",
        "GPUs are optimized to parallelize the feature generation process and hence will give better results when multiple images are passed instead of just one image.\n",
        "The opportunity to improve can be seen based on GPU Utilization. Low GPU Utilization indicates an opportunity to further improve the througput.\n",
        "\n",
        "\n",
        "GPU Utilization can be seen using the nvidia-smi command. To update it every half a second\n",
        "\n",
        "    watch -n .5 nvidia-smi\n",
        "    \n",
        "To pool the GPU utilization every second and dump into a file\n",
        "\n",
        "    nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -f gpu_utilization.csv -l 1\n",
        "    \n",
        "To calculate median GPU Utilization from the file generated\n",
        "\n",
        "    sort -n gpu_utilization.csv | datamash median 1\n",
        "\n",
        "|Model |Time second (sec) | batch_size | % GPU Utilization | Implementation|\n",
        "|-|-|-| -| - |\n",
        "|Resnet50 | 124  | 1  | 52 | extract_features    |\n",
        "|Resnet50 | 98   | 1  | 72 | ImageDataGenerator |\n",
        "|Resnet50 | 57   | 2  | 81 | ImageDataGenerator |\n",
        "|Resnet50 | 40   | 4  | 88 | ImageDataGenerator |\n",
        "|Resnet50 | 34   | 8  | 94 | ImageDataGenerator |\n",
        "|Resnet50 | 29   | 16 | 97 | ImageDataGenerator |\n",
        "|Resnet50 | 28   | 32 | 97 | ImageDataGenerator |\n",
        "|Resnet50 | 28   | 64 | 98 | ImageDataGenerator |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r9ZHDlrwVqhu"
      },
      "source": [
        "### Some benchmarks on different model architectures to see relative speeds\n",
        "\n",
        "Keeping batch size of 64, benchmarking the different models\n",
        "\n",
        "|Model |items/second |\n",
        "|-|-|\n",
        "| VGG19     | 31.06 |\n",
        "| VGG16     | 28.16 | \n",
        "| Resnet50  | 28.48 | \n",
        "| Inception | 20.07 |\n",
        "| Mobilenet | 13.45 |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Znq55FRFVqhv"
      },
      "source": [
        "Let's save the features as intermediate files to use later."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B8ehEpVvVqhv"
      },
      "source": [
        "filenames = [root_dir + '/' + s for s in generator.filenames]"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "code_folding": [],
        "id": "o9JXEaJoVqhv"
      },
      "source": [
        "pickle.dump(generator.classes, open('class_ids-caltech101.pickle',\n",
        "                                    'wb'))\n",
        "pickle.dump(filenames, open('filenames-caltech101.pickle', 'wb'))\n",
        "pickle.dump(\n",
        "    feature_list,\n",
        "    open('features-caltech101-' + model_architecture + '.pickle', 'wb'))"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7subksE6Vqhv"
      },
      "source": [
        "Let's train a finetuned model as well and save the features for that as well."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9qxIE7YsVqhv"
      },
      "source": [
        "TRAIN_SAMPLES = 8677\n",
        "NUM_CLASSES = 101\n",
        "IMG_WIDTH, IMG_HEIGHT = 224, 224"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aad1Q613Vqhv"
      },
      "source": [
        "train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input,\n",
        "                                   rotation_range=20,\n",
        "                                   width_shift_range=0.2,\n",
        "                                   height_shift_range=0.2,\n",
        "                                   zoom_range=0.2)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z0mn_Za_Vqhv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "998c282b-0ae2-4423-97a2-cde55e188bfd"
      },
      "source": [
        "train_generator = train_datagen.flow_from_directory(root_dir,\n",
        "                                                    target_size=(IMG_WIDTH,\n",
        "                                                                 IMG_HEIGHT),\n",
        "                                                    batch_size=batch_size,\n",
        "                                                    shuffle=True,\n",
        "                                                    seed=12345,\n",
        "                                                    class_mode='categorical')"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 8677 images belonging to 101 classes.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5FLqEIx7Vqhv"
      },
      "source": [
        "def model_maker():\n",
        "    base_model = MobileNet(include_top=False,\n",
        "                           input_shape=(IMG_WIDTH, IMG_HEIGHT, 3))\n",
        "    for layer in base_model.layers[:]:\n",
        "        layer.trainable = False\n",
        "    input = Input(shape=(IMG_WIDTH, IMG_HEIGHT, 3))\n",
        "    custom_model = base_model(input)\n",
        "    custom_model = GlobalAveragePooling2D()(custom_model)\n",
        "    custom_model = Dense(64, activation='relu')(custom_model)\n",
        "    custom_model = Dropout(0.5)(custom_model)\n",
        "    predictions = Dense(NUM_CLASSES, activation='softmax')(custom_model)\n",
        "    return Model(inputs=input, outputs=predictions)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "276RzVO-Vqhv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 547
        },
        "outputId": "1fb9153e-7169-4086-c3fe-11700e40bd72"
      },
      "source": [
        "model_finetuned = model_maker()\n",
        "model_finetuned.compile(loss='categorical_crossentropy',\n",
        "              optimizer=tensorflow.keras.optimizers.Adam(0.001),\n",
        "              metrics=['acc'])\n",
        "model_finetuned.fit(\n",
        "    train_generator,\n",
        "    steps_per_epoch=math.ceil(float(TRAIN_SAMPLES) / batch_size),\n",
        "    epochs=10)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-23-3632d4ca8dfa>:8: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use Model.fit, which supports generators.\n",
            "Epoch 1/10\n",
            "136/136 [==============================] - 105s 772ms/step - loss: 3.8703 - acc: 0.2018\n",
            "Epoch 2/10\n",
            "136/136 [==============================] - 106s 777ms/step - loss: 3.2171 - acc: 0.2950\n",
            "Epoch 3/10\n",
            "136/136 [==============================] - 106s 779ms/step - loss: 2.9426 - acc: 0.3339\n",
            "Epoch 4/10\n",
            " 96/136 [====================>.........] - ETA: 30s - loss: 2.7594 - acc: 0.3592"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-23-3632d4ca8dfa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mtrain_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mceil\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTRAIN_SAMPLES\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     epochs=10)\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/deprecation.py\u001b[0m in \u001b[0;36mnew_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    322\u001b[0m               \u001b[0;34m'in a future version'\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'after %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mdate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m               instructions)\n\u001b[0;32m--> 324\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m     return tf_decorator.make_decorator(\n\u001b[1;32m    326\u001b[0m         \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'deprecated'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1827\u001b[0m         \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1828\u001b[0m         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1829\u001b[0;31m         initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1830\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1831\u001b[0m   @deprecation.deprecated(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    106\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 108\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m     \u001b[0;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1096\u001b[0m                 batch_size=batch_size):\n\u001b[1;32m   1097\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1098\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1099\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1100\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    778\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    779\u001b[0m         \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 780\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    781\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    782\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    805\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    806\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 807\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    808\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    809\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2827\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2828\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2829\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2830\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2831\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1846\u001b[0m                            resource_variable_ops.BaseResourceVariable))],\n\u001b[1;32m   1847\u001b[0m         \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1848\u001b[0;31m         cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[1;32m   1849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1850\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1922\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1923\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1924\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1925\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1926\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    548\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nQQ_ILc9Vqhv"
      },
      "source": [
        "model_finetuned.save('./data/model-finetuned-mobilenet.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sNiH1uVNVqhv"
      },
      "source": [
        "start_time = time.time()\n",
        "feature_list_finetuned = []\n",
        "feature_list_finetuned = model_finetuned.predict_generator(generator, num_epochs)\n",
        "end_time = time.time()\n",
        "\n",
        "for i, features_finetuned in enumerate(feature_list_finetuned):\n",
        "    feature_list_finetuned[i] = features_finetuned / norm(features_finetuned)\n",
        "\n",
        "feature_list = feature_list_finetuned.reshape(num_images, -1)\n",
        "\n",
        "print(\"Num images   = \", len(generator.classes))\n",
        "print(\"Shape of feature_list = \", feature_list.shape)\n",
        "print(\"Time taken in sec = \", end_time - start_time)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-PlpDtIvVqhv"
      },
      "source": [
        "pickle.dump(\n",
        "    feature_list,\n",
        "    open('features-caltech101-resnet-finetuned.pickle', 'wb'))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}