Project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hand Detection and Finger Counting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this project, with the help of openCV, we create a program which detects the hand and counts the number of fingers being held up. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing the required libraries\n",
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import pairwise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating some global variables\n",
    "background=None\n",
    "\n",
    "accumulated_weight = 0.5\n",
    "\n",
    "roi_top = 20\n",
    "roi_bottom = 300\n",
    "roi_right = 300\n",
    "roi_left = 600\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_accum_avg(frame,accumulated_weight):\n",
    "    global background\n",
    "    \n",
    "    if background is None:\n",
    "        background = frame.copy().astype('float')\n",
    "        #Returns None only the first time, when background is None\n",
    "        return None\n",
    "    #Subsequently returns nothing\n",
    "    cv2.accumulateWeighted(frame,background,accumulated_weight) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here, we utilize thresholding to obtain the hand segment from the region of interest (ROI)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def segment(frame, threshold = 25):\n",
    "    \n",
    "    #Compute the absolute difference between the frame and background\n",
    "    diff = cv2.absdiff(background.astype('uint8'),frame)\n",
    "    \n",
    "    ret, thresh = cv2.threshold(diff,threshold,255,cv2.THRESH_BINARY)\n",
    "    \n",
    "    image,contours,hierarchy = cv2.findContours(thresh.copy(),cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\n",
    "    \n",
    "    if len(contours)==0: return None\n",
    "    \n",
    "    #The largest external countor in the ROI is expected to be the hand\n",
    "    handSegment = max(contours,key=cv2.contourArea)\n",
    "    \n",
    "    return thresh, handSegment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Counting Fingers via Convex Hull**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A convex hull is the smallest convex polygon that contains all the points in a frame, as shown in the figure below:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](Capture.PNG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We obtain the convex hull of the thresholded image of our hand:\n",
    "\n",
    "![](download.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To count the number of fingers being held up, we begin by: \n",
    "\n",
    "I) Determine the most extreme top, bottom, left, and right points of the convex hull perimeter.\n",
    "\n",
    "II) Compute their intersection and estimate this as the center of the hand.\n",
    "\n",
    "III) Compute the distance $d$ of the point furthest away from this center.\n",
    "\n",
    "IV) Draw a circle centered at the intersection, with radius $\\gamma d$, where $0 \\leq \\gamma \\leq 1$.\n",
    "\n",
    "The idea is this: Any points on the convex hull which lie outside this circle and are far from the bottom of the frame (so we don't erroneously detect our wrist) are likely to be tips of extended fingers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fingerCount(thresh,hand,gamma):\n",
    "    \n",
    "    #Compute the convex hull of the hand\n",
    "    hull = cv2.convexHull(hand)\n",
    "    \n",
    "    #Find the top, bottom, left , and right.\n",
    "    #Then make sure they are in tuple format\n",
    "    top    = tuple(hull[hull[:, :, 1].argmin()][0])\n",
    "    bottom = tuple(hull[hull[:, :, 1].argmax()][0])\n",
    "    left   = tuple(hull[hull[:, :, 0].argmin()][0])\n",
    "    right  = tuple(hull[hull[:, :, 0].argmax()][0])\n",
    "\n",
    "    #We compute the coordinates of the center\n",
    "    cX = (left[0] + right[0]) // 2\n",
    "    cY = (top[1] + bottom[1]) // 2\n",
    "    dist = pairwise.euclidean_distances([(cX,cY)], Y=[left,right,top,bottom])[0]\n",
    "    max_dist = dist.max()\n",
    "    \n",
    "    radius = int(gamma*max_dist)\n",
    "    circumference = 2*np.pi*radius\n",
    "    \n",
    "    #Defining the region of interest\n",
    "    circ_roi = np.zeros(thresh.shape[:2], dtype=\"uint8\")\n",
    "    \n",
    "    cv2.circle(circ_roi,(cX,cY),radius,255,10)\n",
    "    \n",
    "    circ_roi = cv2.bitwise_and(thresh,thresh,mask=circ_roi)\n",
    "    \n",
    "    #Getting the contours in our ROI\n",
    "    image, contours, hierarchy = cv2.findContours(circ_roi.copy(),cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)\n",
    "    \n",
    "    #We count the points that lie outside the circle\n",
    "    count = 0\n",
    "    \n",
    "    for a in contours:\n",
    "        \n",
    "        #Getting the bounding box of the contour\n",
    "        (x,y,w,h) = cv2.boundingRect(a)\n",
    "        \n",
    "        not_wrist = (cY+(cY*0.25)) > (y+h)\n",
    "        \n",
    "        within = ((circumference*0.25)>a.shape[0])\n",
    "        \n",
    "        if not_wrist and within:\n",
    "            count+=1\n",
    "            \n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.41421356, 2.82842712]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairwise.euclidean_distances([(3,4)],[(4,5),(5,6)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Running the Program**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "cam = cv2.VideoCapture(0)\n",
    "\n",
    "num_frames = 0\n",
    "\n",
    "width = int(cam.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
    "height = int(cam.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
    "\n",
    "#Creating the video writer object\n",
    "writer = cv2.VideoWriter('hand_capture.mp4', cv2.VideoWriter_fourcc(*'VIDX'),25, (width, height))\n",
    "\n",
    "while True:\n",
    "    \n",
    "    ret, frame = cam.read()\n",
    "    \n",
    "    frame = cv2.flip(frame,1)\n",
    "    \n",
    "    frame_copy = frame.copy()\n",
    "    \n",
    "    roi = frame[roi_top:roi_bottom,roi_right:roi_left]\n",
    "    \n",
    "    #Converting our image to grayscale\n",
    "    grayscale = cv2.cvtColor(roi,cv2.COLOR_BGR2GRAY)\n",
    "    \n",
    "    #Blurring the image using a Gaussian kernel\n",
    "    grayscale = cv2.GaussianBlur(grayscale,(7,7),0)\n",
    "    \n",
    "    #Getting the background from the first 60 frames\n",
    "    if num_frames < 60:\n",
    "        calc_accum_avg(grayscale, accumulated_weight)\n",
    "        \n",
    "        if num_frames <= 59:\n",
    "            cv2.putText(frame_copy, 'Getting background...', (200,400),cv2.FONT_HERSHEY_DUPLEX,1,(0,255,0),2)\n",
    "            cv2.imshow('Finger count',frame_copy)\n",
    "    else:\n",
    "        hand = segment(grayscale)\n",
    "        \n",
    "        if hand:\n",
    "            thresh, handSegment = hand\n",
    "         \n",
    "            #We draw contours around the hand during the live stream\n",
    "            cv2.drawContours(frame_copy,[handSegment+(roi_right,roi_top)],-1,(255,0,0),5)\n",
    "        \n",
    "            num_fingers = fingerCount(thresh, handSegment, 0.85)\n",
    "        \n",
    "            cv2.putText(frame_copy, str(num_fingers),(70,45),cv2.FONT_HERSHEY_DUPLEX,1,(0,0,255),2)\n",
    "        \n",
    "            cv2.imshow('Thresholded', thresh)\n",
    "    \n",
    "    #Writing the video to file\n",
    "    writer.write(frame_copy)\n",
    "    #Drawing the rectangular ROI\n",
    "    cv2.rectangle(frame_copy,(roi_left,roi_top),(roi_right,roi_bottom),(0,0,255),5)\n",
    "    \n",
    "    num_frames += 1\n",
    "    \n",
    "    cv2.imshow('Finger count',frame_copy)\n",
    "    \n",
    "    k = cv2.waitKey(1) & 0xFF\n",
    "    \n",
    "    if k==27:\n",
    "        break\n",
    "    \n",
    "cam.release()\n",
    "writer.release()\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}