Automatic_Video_Annotation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "AVA Project Batch-8.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "q_t42vB_OWHO"
      },
      "source": [
        "  %tensorflow_version 2.x\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dfXOx8I4Oab8"
      },
      "source": [
        "import os\n",
        " \n",
        "import scipy.io\n",
        "import scipy.misc\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import PIL\n",
        "import struct\n",
        "import cv2\n",
        "from numpy import expand_dims\n",
        "import tensorflow as tf\n",
        "from skimage.transform import resize\n",
        "from keras import backend as K\n",
        "from keras.layers import Input, Lambda, Conv2D, BatchNormalization, LeakyReLU, ZeroPadding2D, UpSampling2D\n",
        "from keras.models import load_model, Model\n",
        "from keras.layers.merge import add, concatenate\n",
        "from keras.preprocessing.image import load_img\n",
        "from keras.preprocessing.image import img_to_array\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.pyplot import imshow\n",
        "from matplotlib.patches import Rectangle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a3OwvXI7Oft8"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vP4Y3A4oOjT6"
      },
      "source": [
        "class WeightReader:\n",
        "    def __init__(self, weight_file):\n",
        "        with open(weight_file, 'rb') as w_f:\n",
        "            major,    = struct.unpack('i', w_f.read(4))\n",
        "            minor,    = struct.unpack('i', w_f.read(4))\n",
        "            revision, = struct.unpack('i', w_f.read(4))\n",
        "\n",
        "            if (major*10 + minor) >= 2 and major < 1000 and minor < 1000:\n",
        "                w_f.read(8)\n",
        "            else:\n",
        "                w_f.read(4)\n",
        "\n",
        "            transpose = (major > 1000) or (minor > 1000)\n",
        "            \n",
        "            binary = w_f.read()\n",
        "\n",
        "        self.offset = 0\n",
        "        self.all_weights = np.frombuffer(binary, dtype='float32')\n",
        "        \n",
        "    def read_bytes(self, size):\n",
        "        self.offset = self.offset + size\n",
        "        return self.all_weights[self.offset-size:self.offset]\n",
        "\n",
        "    def load_weights(self, model):\n",
        "        for i in range(106):\n",
        "            try:\n",
        "                conv_layer = model.get_layer('conv_' + str(i))\n",
        "                print(\"loading weights of convolution #\" + str(i))\n",
        "\n",
        "                if i not in [81, 93, 105]:\n",
        "                    norm_layer = model.get_layer('bnorm_' + str(i))\n",
        "\n",
        "                    size = np.prod(norm_layer.get_weights()[0].shape)\n",
        "\n",
        "                    beta  = self.read_bytes(size) # bias\n",
        "                    gamma = self.read_bytes(size) # scale\n",
        "                    mean  = self.read_bytes(size) # mean\n",
        "                    var   = self.read_bytes(size) # variance            \n",
        "\n",
        "                    weights = norm_layer.set_weights([gamma, beta, mean, var])  \n",
        "\n",
        "                if len(conv_layer.get_weights()) > 1:\n",
        "                    bias   = self.read_bytes(np.prod(conv_layer.get_weights()[1].shape))\n",
        "                    kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n",
        "                    \n",
        "                    kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n",
        "                    kernel = kernel.transpose([2,3,1,0])\n",
        "                    conv_layer.set_weights([kernel, bias])\n",
        "                else:\n",
        "                    kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n",
        "                    kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n",
        "                    kernel = kernel.transpose([2,3,1,0])\n",
        "                    conv_layer.set_weights([kernel])\n",
        "            except ValueError:\n",
        "                print(\"no convolution #\" + str(i))     \n",
        "    \n",
        "    def reset(self):\n",
        "        self.offset = 0\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "70qv4mlhOoyU"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qla27WfWOrTh"
      },
      "source": [
        "def _conv_block(inp, convs, skip=True):\n",
        "    x = inp\n",
        "    count = 0\n",
        "    \n",
        "    for conv in convs:\n",
        "        if count == (len(convs) - 2) and skip:\n",
        "            skip_connection = x\n",
        "        count += 1\n",
        "        \n",
        "        \n",
        "        if conv['stride'] > 1: x = ZeroPadding2D(((1,0),(1,0)))(x) # peculiar padding as darknet prefer left and top\n",
        "        x = Conv2D(conv['filter'], \n",
        "                   conv['kernel'], \n",
        "                   strides=conv['stride'], \n",
        "                   padding='valid' if conv['stride'] > 1 else 'same', # peculiar padding as darknet prefer left and top\n",
        "                   name='conv_' + str(conv['layer_idx']), \n",
        "                   use_bias=False if conv['bnorm'] else True)(x)\n",
        "        if conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x)\n",
        "        if conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x)\n",
        "\n",
        "    return add([skip_connection, x]) if skip else x\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UhCB9aIHOwAB"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ykfct7jxOxzl"
      },
      "source": [
        "def make_yolov3_model():\n",
        "    input_image = Input(shape=(None, None, 3))\n",
        "\n",
        "    # Layer  0 => 4\n",
        "    x = _conv_block(input_image, [{'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0},\n",
        "                                  {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1},\n",
        "                                  {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2},\n",
        "                                  {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}])\n",
        "\n",
        "    # Layer  5 => 8\n",
        "    x = _conv_block(x, [{'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5},\n",
        "                        {'filter':  64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6},\n",
        "                        {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}])\n",
        "\n",
        "    # Layer  9 => 11\n",
        "    x = _conv_block(x, [{'filter':  64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9},\n",
        "                        {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}])\n",
        "\n",
        "    # Layer 12 => 15\n",
        "    x = _conv_block(x, [{'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12},\n",
        "                        {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13},\n",
        "                        {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}])\n",
        "\n",
        "    # Layer 16 => 36\n",
        "    for i in range(7):\n",
        "        x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3},\n",
        "                            {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}])\n",
        "        \n",
        "    skip_36 = x\n",
        "        \n",
        "    # Layer 37 => 40\n",
        "    x = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37},\n",
        "                        {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38},\n",
        "                        {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}])\n",
        "\n",
        "    # Layer 41 => 61\n",
        "    for i in range(7):\n",
        "        x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3},\n",
        "                            {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}])\n",
        "        \n",
        "    skip_61 = x\n",
        "        \n",
        "    # Layer 62 => 65\n",
        "    x = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62},\n",
        "                        {'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63},\n",
        "                        {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}])\n",
        "\n",
        "    # Layer 66 => 74\n",
        "    for i in range(3):\n",
        "        x = _conv_block(x, [{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3},\n",
        "                            {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}])\n",
        "        \n",
        "    # Layer 75 => 79\n",
        "    x = _conv_block(x, [{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75},\n",
        "                        {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76},\n",
        "                        {'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77},\n",
        "                        {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78},\n",
        "                        {'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79}], skip=False)\n",
        "\n",
        "    # Layer 80 => 82\n",
        "    yolo_82 = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 80},\n",
        "                              {'filter':  255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81}], skip=False)\n",
        "\n",
        "    # Layer 83 => 86\n",
        "    x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84}], skip=False)\n",
        "    x = UpSampling2D(2)(x)\n",
        "    x = concatenate([x, skip_61])\n",
        "\n",
        "    # Layer 87 => 91\n",
        "    x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87},\n",
        "                        {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88},\n",
        "                        {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89},\n",
        "                        {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90},\n",
        "                        {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91}], skip=False)\n",
        "\n",
        "    # Layer 92 => 94\n",
        "    yolo_94 = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 92},\n",
        "                              {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93}], skip=False)\n",
        "\n",
        "    # Layer 95 => 98\n",
        "    x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True,   'layer_idx': 96}], skip=False)\n",
        "    x = UpSampling2D(2)(x)\n",
        "    x = concatenate([x, skip_36])\n",
        "\n",
        "    # Layer 99 => 106\n",
        "    yolo_106 = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 99},\n",
        "                               {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 100},\n",
        "                               {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 101},\n",
        "                               {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 102},\n",
        "                               {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 103},\n",
        "                               {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 104},\n",
        "                               {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105}], skip=False)\n",
        "\n",
        "\n",
        "    model = Model(input_image, [yolo_82, yolo_94, yolo_106])    \n",
        "    return model\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oM1eKcglO3-1"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NyPasgR5O6Ze"
      },
      "source": [
        "yolov3 = make_yolov3_model()\n",
        "\n",
        "# load the weights trained on COCO into the model\n",
        "weight_reader = WeightReader('/content/drive/My Drive/yolov3.weights')\n",
        "weight_reader.load_weights(yolov3)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QT2sgWXePCru"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "REQxiCcDPC21"
      },
      "source": [
        "net_h, net_w = 416, 416\n",
        "obj_thresh, nms_thresh = 0.5, 0.45\n",
        "anchors = [[116,90,  156,198,  373,326],  [30,61, 62,45,  59,119], [10,13,  16,30,  33,23]]\n",
        "labels = [\"person\", \"bicycle\", \"car\", \"motorbike\", \"aeroplane\", \"bus\", \"train\", \"truck\", \\\n",
        "              \"boat\", \"traffic light\", \"fire hydrant\", \"stop sign\", \"parking meter\", \"bench\", \\\n",
        "              \"bird\", \"cat\", \"dog\", \"horse\", \"sheep\", \"cow\", \"elephant\", \"bear\", \"zebra\", \"giraffe\", \\\n",
        "              \"backpack\", \"umbrella\", \"handbag\", \"tie\", \"suitcase\", \"frisbee\", \"skis\", \"snowboard\", \\\n",
        "              \"sports ball\", \"kite\", \"baseball bat\", \"baseball glove\", \"skateboard\", \"surfboard\", \\\n",
        "              \"tennis racket\", \"bottle\", \"wine glass\", \"cup\", \"fork\", \"knife\", \"spoon\", \"bowl\", \"banana\", \\\n",
        "              \"apple\", \"sandwich\", \"orange\", \"broccoli\", \"carrot\", \"hot dog\", \"pizza\", \"donut\", \"cake\", \\\n",
        "              \"chair\", \"sofa\", \"pottedplant\", \"bed\", \"diningtable\", \"toilet\", \"tvmonitor\", \"laptop\", \"mouse\", \\\n",
        "              \"remote\", \"keyboard\", \"cell phone\", \"microwave\", \"oven\", \"toaster\", \"sink\", \"refrigerator\", \\\n",
        "              \"book\", \"clock\", \"vase\", \"scissors\", \"teddy bear\", \"hair drier\", \"toothbrush\"]\n",
        "\n",
        "# make the yolov3 model to predict 80 classes on coco\n",
        "\n",
        "#yolov3 = make_yolov3_model()\n",
        "\n",
        "# load the weights trained on COCO into the model\n",
        "#weight_reader = WeightReader('/content/drive/My Drive/yolov3.weights')\n",
        "#weight_reader.load_weights(yolov3)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4XPlQc3DPIB0"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SX3xmF94PMxX"
      },
      "source": [
        "yolov3.save('yolov3.h5')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "152vfZdwPNgf"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EJuzx7DwPQLZ"
      },
      "source": [
        "yolov3 = load_model('yolov3.h5')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OQwM9TaZPVqU"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bz_9dWkSPV4T"
      },
      "source": [
        "from numpy import expand_dims\n",
        "def load_image_pixels(filename, shape):\n",
        "    # load the image to get its shape\n",
        "    image = load_img(filename)\n",
        "    width, height = image.size\n",
        "   \n",
        "    # load the image with the required size\n",
        "    image = load_img(filename, target_size=shape)\n",
        "    \n",
        "    # convert to numpy array\n",
        "    \n",
        "    image = img_to_array(image)\n",
        "    # scale pixel values to [0, 1]\n",
        "    image = image.astype('float32')\n",
        "    image /= 255.0\n",
        "    # add a dimension so that we have one sample\n",
        "    image = expand_dims(image, 0)\n",
        "    return image, width, height\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lz9WBwnaPeQM"
      },
      "source": [
        "class BoundBox:\n",
        "    def __init__(self, xmin, ymin, xmax, ymax, objness = None, classes = None):\n",
        "        self.xmin = xmin\n",
        "        self.ymin = ymin\n",
        "        self.xmax = xmax\n",
        "        self.ymax = ymax\n",
        "        \n",
        "        self.objness = objness\n",
        "        self.classes = classes\n",
        "\n",
        "        self.label = -1\n",
        "        self.score = -1\n",
        "\n",
        "    def get_label(self):\n",
        "        if self.label == -1:\n",
        "            self.label = np.argmax(self.classes)\n",
        "        \n",
        "        return self.label\n",
        "    \n",
        "    def get_score(self):\n",
        "        if self.score == -1:\n",
        "            self.score = self.classes[self.get_label()]\n",
        "            \n",
        "        return self.score\n",
        "\n",
        "def _sigmoid(x):\n",
        "    return 1. / (1. + np.exp(-x))\n",
        "\n",
        "def _interval_overlap(interval_a, interval_b):\n",
        "    x1, x2 = interval_a\n",
        "    x3, x4 = interval_b\n",
        "\n",
        "    if x3 < x1:\n",
        "        if x4 < x1:\n",
        "            return 0\n",
        "        else:\n",
        "            return min(x2,x4) - x1\n",
        "    else:\n",
        "        if x2 < x3:\n",
        "             return 0\n",
        "        else:\n",
        "            return min(x2,x4) - x3 \n",
        "def bbox_iou(box1, box2):\n",
        "    intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])\n",
        "    intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax])\n",
        "    \n",
        "    intersect = intersect_w * intersect_h\n",
        "\n",
        "    w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin\n",
        "    w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin\n",
        "    \n",
        "    union = w1*h1 + w2*h2 - intersect\n",
        "    \n",
        "    return float(intersect) / union\n",
        "\n",
        "def do_nms(boxes, nms_thresh):\n",
        "    if len(boxes) > 0:\n",
        "        nb_class = len(boxes[0].classes)\n",
        "    else:\n",
        "        return\n",
        "        \n",
        "    for c in range(nb_class):\n",
        "        sorted_indices = np.argsort([-box.classes[c] for box in boxes])\n",
        "\n",
        "        for i in range(len(sorted_indices)):\n",
        "            index_i = sorted_indices[i]\n",
        "\n",
        "            if boxes[index_i].classes[c] == 0: continue\n",
        "\n",
        "            for j in range(i+1, len(sorted_indices)):\n",
        "                index_j = sorted_indices[j]\n",
        "\n",
        "                if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh:\n",
        "                    boxes[index_j].classes[c] = 0\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NcfGhKdEPj3q"
      },
      "source": [
        "#decode_netout() that will take each one of the NumPy arrays, one at a time, \n",
        "#and decode the candidate bounding boxes and class predictions\n",
        "def decode_netout(netout, anchors, obj_thresh,  net_h, net_w):\n",
        "    grid_h, grid_w = netout.shape[:2]\n",
        "    nb_box = 3\n",
        "    netout = netout.reshape((grid_h, grid_w, nb_box, -1))\n",
        "    nb_class = netout.shape[-1] - 5\n",
        "\n",
        "    boxes = []\n",
        "\n",
        "    netout[..., :2]  = _sigmoid(netout[..., :2])\n",
        "    netout[..., 4:]  = _sigmoid(netout[..., 4:])\n",
        "    netout[..., 5:]  = netout[..., 4][..., np.newaxis] * netout[..., 5:]\n",
        "    netout[..., 5:] *= netout[..., 5:] > obj_thresh\n",
        "\n",
        "    for i in range(grid_h*grid_w):\n",
        "        row = i / grid_w\n",
        "        col = i % grid_w\n",
        "        \n",
        "        for b in range(nb_box):\n",
        "            # 4th element is objectness score\n",
        "            objectness = netout[int(row)][int(col)][b][4]\n",
        "            #objectness = netout[..., :4]\n",
        "            \n",
        "            if(objectness.all() <= obj_thresh): continue\n",
        "            \n",
        "            # first 4 elements are x, y, w, and h\n",
        "            x, y, w, h = netout[int(row)][int(col)][b][:4]\n",
        "\n",
        "            x = (col + x) / grid_w # center position, unit: image width\n",
        "            y = (row + y) / grid_h # center position, unit: image height\n",
        "            w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width\n",
        "            h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height  \n",
        "            \n",
        "            # last elements are class probabilities\n",
        "            classes = netout[int(row)][col][b][5:]\n",
        "            \n",
        "            box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes)\n",
        "            #box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, None, classes)\n",
        "\n",
        "            boxes.append(box)\n",
        "\n",
        "    return boxes\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5o4g64NCPnlJ"
      },
      "source": [
        "#will allow plotting the original image and draw the bounding boxes, hopefully detecting real objects.\n",
        "# correct the sizes of the bounding boxes for the shape of the image\n",
        "#correct_yolo_boxes(boxes, image_h, image_w, input_h, input_w)\n",
        "def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w):\n",
        "    if (float(net_w)/image_w) < (float(net_h)/image_h):\n",
        "        new_w = net_w\n",
        "        new_h = (image_h*net_w)/image_w\n",
        "    else:\n",
        "        new_h = net_w\n",
        "        new_w = (image_w*net_h)/image_h\n",
        "        \n",
        "    for i in range(len(boxes)):\n",
        "        x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w\n",
        "        y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h\n",
        "        \n",
        "        boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w)\n",
        "        boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w)\n",
        "        boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h)\n",
        "        boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3dYDtLKswPUC"
      },
      "source": [
        "def get_boxes(boxes, labels, thresh):\n",
        "  v_boxes, v_labels, v_scores = list(), list(), list() \n",
        "  # enumerate all boxes\n",
        "  for box in boxes:\n",
        "    # enumerate all possible labels\n",
        "    for i in range(len(labels)):\n",
        "      # check if the threshold for this label is high enough \n",
        "      if box.classes[i] > thresh:\n",
        "        v_boxes.append(box) \n",
        "        v_labels.append(labels[i]) \n",
        "        v_scores.append(box.classes[i]*100)\n",
        "        # don't break, many labels may trigger for one box \n",
        "  return v_boxes, v_labels, v_scores\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wLBi0splPrE5"
      },
      "source": [
        "from matplotlib.patches import Rectangle\n",
        "def draw_boxes(filename, v_boxes, v_labels, v_scores):\n",
        "    # load the image\n",
        "    #print(filename)\n",
        "    data1=cv2.imread(filename)\n",
        "    #data = plt.imread(filename)\n",
        "    #data = filename\n",
        "    # plot the image\n",
        "    #plt.imshow(data)\n",
        "    #plt.xticks([1]),plt.yticks([1])\n",
        "    # get the context for drawing boxes\n",
        "    #ax = plt.gca()\n",
        "    # plot each box\n",
        "    for i in range(len(v_boxes)):\n",
        "        box = v_boxes[i]\n",
        "        # get coordinates\n",
        "        y1, x1, y2, x2 = box.ymin, box.xmin, box.ymax, box.xmax\n",
        "        # calculate width and height of the box\n",
        "        width, height = x2 - x1, y2 - y1\n",
        "        # create the shape\n",
        "        #rect = Rectangle((x1, y1-30), width-100, height, fill=False, color='red')\n",
        "        # draw the box\n",
        "        #ax.add_patch(rect)\n",
        "        # draw text and score in top left corner\n",
        "        cv2.rectangle(data1,(x1,y1),(x1+width,y1+height),0,1)\n",
        "                           #Addition\n",
        "        label = \"%.1f %s (%.3f)\" % ((i+1),v_labels[i], v_scores[i])\n",
        "        cv2.putText(data1,label,(x1,y1+30),cv2.FONT_HERSHEY_COMPLEX_SMALL,2,1,2)\n",
        "        #plt.text(x1, y1, label, color='red')\n",
        "    #plt.savefig(\"krishna.jpg\",dpi=200)\n",
        "    cv2.imwrite('krishna.jpg',data1)\n",
        "    # show the plot\n",
        "    #plt.show()\n",
        "    return data1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q1ZjtEiax_x4"
      },
      "source": [
        "# define the expected input shape for the model input_w, input_h = 416, 416\n",
        "# define our new photo\n",
        "photo_filename = '/content/photo.jpg' \n",
        "# load and prepare image\n",
        "\n",
        "image, image_w, image_h = load_image_pixels(photo_filename, (net_w, net_h)) \n",
        "print(image.shape)\n",
        "print(image_w,image_h) \n",
        "# make prediction\n",
        " \n",
        "yolos = yolov3.predict(image) \n",
        "#print(yolos)\n",
        "# summarize the shape of the list of arrays \n",
        "print([a.shape for a in yolos])\n",
        "\n",
        "# define the anchors\n",
        "anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]]\n",
        "# define the probability threshold for detected objects \n",
        "class_threshold = 0.6\n",
        "boxes = list()\n",
        "\n",
        "for i in range(len(yolos)):\n",
        "  # decode the output of the network\n",
        "  boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, net_h, net_w)\n",
        "  # correct the sizes of the bounding boxes \n",
        "correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w)\n",
        "\n",
        "# suppress non-maximal boxes \n",
        "do_nms(boxes, nms_thresh)\n",
        "\n",
        "# get the details of the detected objects\n",
        "v_boxes, v_labels, v_scores = get_boxes(boxes, labels, class_threshold) \n",
        "# summarize what we found\n",
        "for i in range(len(v_boxes)): \n",
        "  print(v_labels[i], v_scores[i])\n",
        "# draw what we found\n",
        "\n",
        "draw_boxes(photo_filename, v_boxes, v_labels, v_scores)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-eUjgHzuPuwi"
      },
      "source": [
        "import cv2\n",
        "from matplotlib import pyplot as plt\n",
        "videocap = cv2.VideoCapture(\"/content/drive/My Drive/WhatsApp Video 2020-07-27 at 2.21.59 PM.mp4\");\n",
        "success, image = videocap.read(); \n",
        "count=0\n",
        "l=[]\n",
        "success=True\n",
        "height, width, layers = image.shape \n",
        "size = (width,height)\n",
        "while success:\n",
        "  #cv2.imwrite(\"frame%d.jpg\" % count, image) \n",
        "  l.append(image)\n",
        " \n",
        "  success, image = videocap.read() \n",
        "  print(\"read a new frame:\",success) \n",
        "  if success==False:\n",
        "    break\n",
        "\n",
        "  count+=1 \n",
        "  cv2.waitKey(0)\n",
        "  cv2.destroyAllWindows()\n",
        "out = cv2.VideoWriter('project.avi',cv2.VideoWriter_fourcc(*'DIVX'), 15, size) \n",
        "for i in range(len(l)):\n",
        "  out.write(l[i]) \n",
        "out.release()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4G05NO5BPx2O"
      },
      "source": [
        "import cv2\n",
        "from keras.preprocessing.image import array_to_img \n",
        "from numpy import expand_dims\n",
        "from matplotlib import pyplot as plt\n",
        "input_w, input_h = 416, 416\n",
        "videocap = cv2.VideoCapture(\"/content/drive/My Drive/WhatsApp Video 2020-07-27 at 2.21.59 PM.mp4\"); \n",
        "success, image = videocap.read()\n",
        "count=1 \n",
        "l=[]\n",
        "success=True \n",
        "while success:\n",
        "  success, image = videocap.read() \n",
        "  if success==False:\n",
        "    break\n",
        "  cv2.imwrite('photo.jpg',image) \n",
        "  cv2.imwrite('photo1.jpg',image)\n",
        "  #plt.imshow(image) #plt.savefig('photo.jpg',dpi=200) #plt.savefig('photo1.jpg',dpi=200) \n",
        "  filename='photo.jpg' \n",
        "  filename1='photo1.jpg'\n",
        "  # load and prepare image\n",
        "  image, image_w, image_h = load_image_pixels(filename, (net_w, net_w)) \n",
        "  # make prediction\n",
        "  size=(image_w,image_h) \n",
        "  yolos = yolov3.predict(image) \n",
        "  #print(yolos)\n",
        "  # summarize the shape of the list of arrays print([a.shape for a in yolos])\n",
        "  # define the anchors\n",
        "  anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]]\n",
        "  \n",
        "  # define the probability threshold for detected objects \n",
        "  class_threshold = 0.6\n",
        "  boxes = list()\n",
        "\n",
        "  for i in range(len(yolos)):\n",
        "    # decode the output of the network\n",
        "    boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, net_h, net_w)\n",
        "\n",
        "  # correct the sizes of the bounding boxes \n",
        "  correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w)\n",
        "\n",
        "  # suppress non-maximal boxes \n",
        "  do_nms(boxes, nms_thresh)\n",
        "\n",
        "  # get the details of the detected objects\n",
        "  v_boxes, v_labels, v_scores = get_boxes(boxes, labels, class_threshold) \n",
        "  # summarize what we found\n",
        "  #for i in range(len(v_boxes)): #print(v_labels[i], v_scores[i])\n",
        "  # draw what we found\n",
        "  image=draw_boxes(filename1, v_boxes, v_labels, v_scores) \n",
        "  #image=cv2.imread('krishna.jpg') \n",
        "  #image=plt.imread('krishna.jpg')\n",
        "  #plt.imshow(image) \n",
        "  #plt.xticks([1]),plt.yticks([1]) \n",
        "  #plt.show()\n",
        "\n",
        "  l.append(image) \n",
        "  #print(\"fine\")\n",
        "out = cv2.VideoWriter('output.avi',cv2.VideoWriter_fourcc(*'DIVX'), 15, size) \n",
        "for i in range(len(l)):\n",
        "  out.write(l[i]) \n",
        "out.release() \n",
        "print(\"finish\")\n",
        " \n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}