checkpoint5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Python Hand Gesture Recognition (Checkpoint 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This checkpoint is at the end of Objective 5.\n",
    "\n",
    "At this point, your code should be fully functional -- it'll recognize waving, pointing, a peace sign (scissors), and a fist (rock). Congrats on making it here!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Header: Importing libraries and creating global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import cv2\n",
    "\n",
    "# Hold the background frame for background subtraction.\n",
    "background = None\n",
    "# Hold the hand's data so all its details are in one place.\n",
    "hand = None\n",
    "# Variables to count how many frames have passed and to set the size of the window.\n",
    "frames_elapsed = 0\n",
    "FRAME_HEIGHT = 200\n",
    "FRAME_WIDTH = 300\n",
    "# Humans come in a ton of beautiful shades and colors.\n",
    "# Try editing these if your program has trouble recognizing your skin tone.\n",
    "CALIBRATION_TIME = 30\n",
    "BG_WEIGHT = 0.5\n",
    "OBJ_THRESHOLD = 18"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HandData: A class to hold all the hand's details and flags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HandData:\n",
    "    top = (0,0)\n",
    "    bottom = (0,0)\n",
    "    left = (0,0)\n",
    "    right = (0,0)\n",
    "    centerX = 0\n",
    "    prevCenterX = 0\n",
    "    isInFrame = False\n",
    "    isWaving = False\n",
    "    fingers = None\n",
    "    gestureList = []\n",
    "    \n",
    "    def __init__(self, top, bottom, left, right, centerX):\n",
    "        self.top = top\n",
    "        self.bottom = bottom\n",
    "        self.left = left\n",
    "        self.right = right\n",
    "        self.centerX = centerX\n",
    "        self.prevCenterX = 0\n",
    "        isInFrame = False\n",
    "        isWaving = False\n",
    "        \n",
    "    def update(self, top, bottom, left, right):\n",
    "        self.top = top\n",
    "        self.bottom = bottom\n",
    "        self.left = left\n",
    "        self.right = right\n",
    "        \n",
    "    def check_for_waving(self, centerX):\n",
    "        self.prevCenterX = self.centerX\n",
    "        self.centerX = centerX\n",
    "        \n",
    "        if abs(self.centerX - self.prevCenterX > 3):\n",
    "            self.isWaving = True\n",
    "        else:\n",
    "            self.isWaving = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### write_on_image(): Write info related to the hand gesture and outline the region of interest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here we take the current frame, the number of frames elapsed, and how many fingers we've detected\n",
    "# so we can print on the screen which gesture is happening (or if the camera is calibrating).\n",
    "def write_on_image(frame):\n",
    "    text = \"Searching...\"\n",
    "\n",
    "    if frames_elapsed < CALIBRATION_TIME:\n",
    "        text = \"Calibrating...\"\n",
    "    elif hand == None or hand.isInFrame == False:\n",
    "        text = \"No hand detected\"\n",
    "    else:\n",
    "        if hand.isWaving:\n",
    "            text = \"Waving\"\n",
    "        elif hand.fingers == 0:\n",
    "            text = \"Rock\"\n",
    "        elif hand.fingers == 1:\n",
    "            text = \"Pointing\"\n",
    "        elif hand.fingers == 2:\n",
    "            text = \"Scissors\"\n",
    "    \n",
    "    cv2.putText(frame, text, (10,20), cv2.FONT_HERSHEY_COMPLEX, 0.4,( 0 , 0 , 0 ),2,cv2.LINE_AA)\n",
    "    cv2.putText(frame, text, (10,20), cv2.FONT_HERSHEY_COMPLEX, 0.4,(255,255,255),1,cv2.LINE_AA)\n",
    "\n",
    "    # Highlight the region of interest.\n",
    "    cv2.rectangle(frame, (region_left, region_top), (region_right, region_bottom), (255,255,255), 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get_region(): Separate the region of interest and preps it for edge detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_region(frame):\n",
    "    # Separate the region of interest from the rest of the frame.\n",
    "    region = frame[region_top:region_bottom, region_left:region_right]\n",
    "    # Make it grayscale so we can detect the edges more easily.\n",
    "    region = cv2.cvtColor(region, cv2.COLOR_BGR2GRAY)\n",
    "    # Use a Gaussian blur to prevent frame noise from being labeled as an edge.\n",
    "    region = cv2.GaussianBlur(region, (5,5), 0)\n",
    "\n",
    "    return region"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get_average(): Create a weighted average of the background for image differencing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_average(region):\n",
    "    # We have to use the global keyword because we want to edit the global variable.\n",
    "    global background\n",
    "    # If we haven't captured the background yet, make the current region the background.\n",
    "    if background is None:\n",
    "        background = region.copy().astype(\"float\")\n",
    "        return\n",
    "    # Otherwise, add this captured frame to the average of the backgrounds.\n",
    "    cv2.accumulateWeighted(region, background, BG_WEIGHT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### segment(): Use image differencing to separate the hand from the background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here we use differencing to separate the background from the object of interest.\n",
    "def segment(region):\n",
    "    global hand\n",
    "    # Find the absolute difference between the background and the current frame.\n",
    "    diff = cv2.absdiff(background.astype(np.uint8), region)\n",
    "\n",
    "    # Threshold that region with a strict 0 or 1 ruling so only the foreground remains.\n",
    "    thresholded_region = cv2.threshold(diff, OBJ_THRESHOLD, 255, cv2.THRESH_BINARY)[1]\n",
    "\n",
    "    # Get the contours of the region, which will return an outline of the hand.\n",
    "    (_, contours, _) = cv2.findContours(thresholded_region.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n",
    "\n",
    "    # If we didn't get anything, there's no hand.\n",
    "    if len(contours) == 0:\n",
    "        if hand is not None:\n",
    "            hand.isInFrame = False\n",
    "        return\n",
    "    # Otherwise return a tuple of the filled hand (thresholded_region), along with the outline (segmented_region).\n",
    "    else:\n",
    "        if hand is not None:\n",
    "            hand.isInFrame = True\n",
    "        segmented_region = max(contours, key = cv2.contourArea)\n",
    "        return (thresholded_region, segmented_region)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get_hand_data(): Find the extremities of the hand and put them in the global hand object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_hand_data(thresholded_image, segmented_image):\n",
    "    global hand\n",
    "    \n",
    "    # Enclose the area around the extremities in a convex hull to connect all outcroppings.\n",
    "    convexHull = cv2.convexHull(segmented_image)\n",
    "    \n",
    "    # Find the extremities for the convex hull and store them as points.\n",
    "    top    = tuple(convexHull[convexHull[:, :, 1].argmin()][0])\n",
    "    bottom = tuple(convexHull[convexHull[:, :, 1].argmax()][0])\n",
    "    left   = tuple(convexHull[convexHull[:, :, 0].argmin()][0])\n",
    "    right  = tuple(convexHull[convexHull[:, :, 0].argmax()][0])\n",
    "    \n",
    "    # Get the center of the palm, so we can check for waving and find the fingers.\n",
    "    centerX = int((left[0] + right[0]) / 2)\n",
    "    \n",
    "    # We put all the info into an object for handy extraction (get it? HANDy?)\n",
    "    if hand == None:\n",
    "        hand = HandData(top, bottom, left, right, centerX)\n",
    "    else:\n",
    "        hand.update(top, bottom, left, right)\n",
    "    \n",
    "    # Only check for waving every 6 frames.\n",
    "    if frames_elapsed % 6 == 0:\n",
    "        hand.check_for_waving(centerX)\n",
    "    \n",
    "    # We count the number of fingers up every frame, but only change hand.fingers if\n",
    "    # 12 frames have passed, to prevent erratic gesture counts.\n",
    "    hand.gestureList.append(count_fingers(thresholded_image))\n",
    "    if frames_elapsed % 12 == 0:\n",
    "        hand.fingers = most_frequent(hand.gestureList)\n",
    "        hand.gestureList.clear()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### count_fingers(): Count the number of fingers using a line intersecting fingertips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_fingers(thresholded_image):\n",
    "    \n",
    "    # Find the height at which we will draw the line to count fingers.\n",
    "    line_height = int(hand.top[1] + (0.2 * (hand.bottom[1] - hand.top[1])))\n",
    "    \n",
    "    # Get the linear region of interest along where the fingers would be.\n",
    "    line = np.zeros(thresholded_image.shape[:2], dtype=int)\n",
    "    \n",
    "    # Draw a line across this region of interest, where the fingers should be.\n",
    "    cv2.line(line, (thresholded_image.shape[1], line_height), (0, line_height), 255, 1)\n",
    "    \n",
    "    # Do a bitwise AND to find where the line intersected the hand -- this is where the fingers are.\n",
    "    line = cv2.bitwise_and(thresholded_image, thresholded_image, mask = line.astype(np.uint8))\n",
    "    \n",
    "    # Get the line's new contours. The contours are basically just little lines formed by gaps \n",
    "    # in the big line across the fingers, so each would be a finger unless it's very wide.\n",
    "    (_, contours, _) = cv2.findContours(line.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n",
    "    \n",
    "    fingers = 0\n",
    "    \n",
    "    # Count the fingers by making sure the contour lines are \"finger-sized\", i.e. not too wide.\n",
    "    # This prevents a \"rock\" gesture from being mistaken for a finger.\n",
    "    for curr in contours:\n",
    "        width = len(curr)\n",
    "        \n",
    "        if width < 3 * abs(hand.right[0] - hand.left[0]) / 4 and width > 5:\n",
    "            fingers += 1\n",
    "    \n",
    "    return fingers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### most_frequent(): Returns the value in a list that appears most frequently"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def most_frequent(input_list):\n",
    "    dict = {}\n",
    "    count = 0\n",
    "    most_freq = 0\n",
    "    \n",
    "    for item in reversed(input_list):\n",
    "        dict[item] = dict.get(item, 0) + 1\n",
    "        if dict[item] >= count :\n",
    "            count, most_freq = dict[item], item\n",
    "    \n",
    "    return most_freq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Main function: Get input from camera and call functions to understand it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Our region of interest will be the top right part of the frame.\n",
    "region_top = 0\n",
    "region_bottom = int(2 * FRAME_HEIGHT / 3)\n",
    "region_left = int(FRAME_WIDTH / 2)\n",
    "region_right = FRAME_WIDTH\n",
    "\n",
    "frames_elapsed = 0\n",
    "\n",
    "capture = cv2.VideoCapture(1)\n",
    "\n",
    "while (True):\n",
    "    # Store the frame from the video capture and resize it to the window size.\n",
    "    ret, frame = capture.read()\n",
    "    frame = cv2.resize(frame, (FRAME_WIDTH, FRAME_HEIGHT))\n",
    "    # Flip the frame over the vertical axis so that it works like a mirror, which is more intuitive to the user.\n",
    "    frame = cv2.flip(frame, 1)\n",
    "    \n",
    "    # Separate the region of interest and prep it for edge detection.\n",
    "    region = get_region(frame)\n",
    "    if frames_elapsed < CALIBRATION_TIME:\n",
    "        get_average(region)\n",
    "    else:\n",
    "        region_pair = segment(region)\n",
    "        if region_pair is not None:\n",
    "            # If we have the regions segmented successfully, show them in another window for the user.\n",
    "            (thresholded_region, segmented_region) = region_pair\n",
    "            cv2.drawContours(region, [segmented_region], -1, (255, 255, 255))\n",
    "            cv2.imshow(\"Segmented Image\", region)\n",
    "            \n",
    "            get_hand_data(thresholded_region, segmented_region)\n",
    "    \n",
    "    # Write the action the hand is doing on the screen, and draw the region of interest.\n",
    "    write_on_image(frame)\n",
    "    # Show the previously captured frame.\n",
    "    cv2.imshow(\"Camera Input\", frame)\n",
    "    frames_elapsed += 1\n",
    "    # Check if user wants to exit.\n",
    "    if (cv2.waitKey(1) & 0xFF == ord('x')):\n",
    "        break\n",
    "\n",
    "# When we exit the loop, we have to stop the capture too.\n",
    "capture.release()\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}