video_coordinates.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/ashish/Documents/UCD/InsightCentre/VirtualEnvironments/multits/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/ashish/Documents/UCD/InsightCentre/VirtualEnvironments/multits/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/ashish/Documents/UCD/InsightCentre/VirtualEnvironments/multits/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/ashish/Documents/UCD/InsightCentre/VirtualEnvironments/multits/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/ashish/Documents/UCD/InsightCentre/VirtualEnvironments/multits/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/ashish/Documents/UCD/InsightCentre/VirtualEnvironments/multits/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import argparse\n",
    "import cv2\n",
    "import time\n",
    "from config_reader import config_reader\n",
    "from processing import extract_parts, draw\n",
    "\n",
    "from model.cmu_model import get_testing_model\n",
    "\n",
    "sys.path.append(os.path.join(os.path.dirname(os.path.abspath('')), \"..\"))\n",
    "\n",
    "currentDT = time.localtime()\n",
    "start_datetime = time.strftime(\"-%m-%d-%H-%M-%S\", currentDT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras_weights_file = \"model.h5\"\n",
    "frame_rate_ratio = 1\n",
    "process_speed = 4\n",
    "ending_frame = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Video input\n",
    "video = \"/home/ashish/Downloads/HumanPose/Binh_VideoDataset_June2019/TimeSeries_Exp4/test.mp4\"\n",
    "video_path = 'videos/'\n",
    "video_file = video\n",
    "\n",
    "# Output location\n",
    "output_path = 'videos/outputs/'\n",
    "output_format = '.mp4'\n",
    "video_output = output_path + video + str(start_datetime) + output_format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vgg normalization (subtracting mean) on input images\n",
    "model = get_testing_model()\n",
    "model.load_weights(keras_weights_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "params, model_params = config_reader()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'use_gpu': 1, 'GPUdeviceNumber': 0, 'modelID': '1', 'octave': 3, 'starting_range': 0.8, 'ending_range': 2.0, 'scale_search': [0.5, 1.0, 1.5, 2.0], 'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5, 'min_num': 4, 'mid_num': 10, 'crop_ratio': 2.5, 'bbox_ratio': 0.25}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'caffemodel': './model/_trained_COCO/pose_iter_440000.caffemodel', 'deployFile': './model/_trained_COCO/pose_deploy.prototxt', 'description': 'COCO Pose56 Two-level Linevec', 'boxsize': 368, 'padValue': 128, 'np': '12', 'stride': 8, 'part_str': ['[nose', 'neck', 'Rsho', 'Relb', 'Rwri', 'Lsho', 'Lelb', 'Lwri', 'Rhip', 'Rkne', 'Rank', 'Lhip', 'Lkne', 'Lank', 'Leye', 'Reye', 'Lear', 'Rear', 'pt19]']}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Video reader\n",
    "cam = cv2.VideoCapture(video_file)\n",
    "input_fps = cam.get(cv2.CAP_PROP_FPS)\n",
    "ret_val, orig_image = cam.read()\n",
    "video_length = int(cam.get(cv2.CAP_PROP_FRAME_COUNT))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def manual_count(handler):\n",
    "    frames = 0\n",
    "    while True:\n",
    "        status, frame = handler.read()\n",
    "        if not status:\n",
    "            break\n",
    "        frames += 1\n",
    "    return frames "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "cam = cv2.VideoCapture(video_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "241"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "manual_count(cam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "if ending_frame is None:\n",
    "    ending_frame = video_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame_rate_ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29.97002997002997"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_fps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_fps = input_fps / frame_rate_ratio\n",
    "fourcc = cv2.VideoWriter_fourcc(*'mp4v')\n",
    "out = cv2.VideoWriter(video_output, fourcc, output_fps, (orig_image.shape[1], orig_image.shape[0]))\n",
    "\n",
    "scale_search = [1, .5, 1.5, 2]  # [.5, 1, 1.5, 2]\n",
    "scale_search = scale_search[0:process_speed]\n",
    "params['scale_search'] = scale_search\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29.97002997002997"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_fps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing frame:  0\n",
      "processing time is 22.05920\n"
     ]
    }
   ],
   "source": [
    "i = 0  # default is 0\n",
    "while(cam.isOpened()) and ret_val is True and i < ending_frame:\n",
    "    if i % frame_rate_ratio == 0:\n",
    "\n",
    "        input_image = cv2.cvtColor(orig_image, cv2.COLOR_RGB2BGR)\n",
    "\n",
    "        tic = time.time()\n",
    "\n",
    "        # generate image with body parts\n",
    "        body_parts, all_peaks, subset, candidate = extract_parts(input_image, params, model, model_params)\n",
    "        # canvas = draw(orig_image, all_peaks, subset, candidate)\n",
    "\n",
    "        print('Processing frame: ', i)\n",
    "        toc = time.time()\n",
    "        print('processing time is %.5f' % (toc - tic))\n",
    "\n",
    "        # out.write(canvas)\n",
    "\n",
    "    ret_val, orig_image = cam.read()\n",
    "\n",
    "    i += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'nose': (483, 167),\n",
       " 'neck': (483, 203),\n",
       " 'right_shoulder': (455, 204),\n",
       " ' right_elbow': (446, 244),\n",
       " 'right_wrist': (446, 285),\n",
       " 'left_shoulder': (510, 203),\n",
       " 'left_elbow': (522, 244),\n",
       " 'left_wrist': (528, 283),\n",
       " 'right_hip': (469, 284),\n",
       " 'right_knee': (463, 352),\n",
       " 'right_ankle': (455, 421),\n",
       " 'left_hip': (505, 283),\n",
       " 'left_knee': (509, 350),\n",
       " 'left_ankle': (516, 420),\n",
       " 'right_eye': (478, 162),\n",
       " 'left_eye': (488, 162),\n",
       " 'right_ear': (471, 168),\n",
       " 'left_ear': (496, 168)}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "body_parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[(483, 167, 0.9262133538722992, 0)],\n",
       " [(483, 203, 0.934542253613472, 1)],\n",
       " [(455, 204, 0.899353638291359, 2)],\n",
       " [(446, 244, 0.8779154121875763, 3)],\n",
       " [(446, 285, 0.8312231004238129, 4)],\n",
       " [(510, 203, 0.8871498703956604, 5)],\n",
       " [(522, 244, 0.8205071836709976, 6)],\n",
       " [(528, 283, 0.7493290826678276, 7)],\n",
       " [(469, 284, 0.7184247523546219, 8)],\n",
       " [(463, 352, 0.8270892202854156, 9)],\n",
       " [(455, 421, 0.8078549355268478, 10)],\n",
       " [(505, 283, 0.7286079674959183, 11)],\n",
       " [(509, 350, 0.8157999813556671, 12)],\n",
       " [(516, 420, 0.7970129102468491, 13)],\n",
       " [(478, 162, 0.9284635633230209, 14)],\n",
       " [(488, 162, 0.9314068257808685, 15)],\n",
       " [(471, 168, 0.841595932841301, 16)],\n",
       " [(496, 168, 0.874936044216156, 17)]]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_peaks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.        ,  1.        ,  2.        ,  3.        ,  4.        ,\n",
       "         5.        ,  6.        ,  7.        ,  8.        ,  9.        ,\n",
       "        10.        , 11.        , 12.        , 13.        , 14.        ,\n",
       "        15.        , 16.        , 17.        , 31.90221745, 18.        ]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[483.        , 167.        ,   0.92621335,   0.        ],\n",
       "       [483.        , 203.        ,   0.93454225,   1.        ],\n",
       "       [455.        , 204.        ,   0.89935364,   2.        ],\n",
       "       [446.        , 244.        ,   0.87791541,   3.        ],\n",
       "       [446.        , 285.        ,   0.8312231 ,   4.        ],\n",
       "       [510.        , 203.        ,   0.88714987,   5.        ],\n",
       "       [522.        , 244.        ,   0.82050718,   6.        ],\n",
       "       [528.        , 283.        ,   0.74932908,   7.        ],\n",
       "       [469.        , 284.        ,   0.71842475,   8.        ],\n",
       "       [463.        , 352.        ,   0.82708922,   9.        ],\n",
       "       [455.        , 421.        ,   0.80785494,  10.        ],\n",
       "       [505.        , 283.        ,   0.72860797,  11.        ],\n",
       "       [509.        , 350.        ,   0.81579998,  12.        ],\n",
       "       [516.        , 420.        ,   0.79701291,  13.        ],\n",
       "       [478.        , 162.        ,   0.92846356,  14.        ],\n",
       "       [488.        , 162.        ,   0.93140683,  15.        ],\n",
       "       [471.        , 168.        ,   0.84159593,  16.        ],\n",
       "       [496.        , 168.        ,   0.87493604,  17.        ]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " body_parts, all_peaks, subset, candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/ashish/Downloads/keras_Realtime_Multi-Person_Pose_Estimation-master'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "caffe_to_keras.py  demo.ipynb\t\t processing.py\tutil.py\r\n",
      "config\t\t   demo_video.py\t __pycache__\tvideo_coordinates.ipynb\r\n",
      "config_reader.py   dump_caffe_layers.py  readme\t\tvideos\r\n",
      "dataset\t\t   LICENSE\t\t README.md\r\n",
      "demo_camera.py\t   model\t\t sample_images\r\n",
      "demo_image.py\t   model.h5\t\t training\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "ename": "error",
     "evalue": "OpenCV(4.1.2) /io/opencv/modules/imgproc/src/color.cpp:182: error: (-215:Assertion failed) !_src.empty() in function 'cvtColor'\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31merror\u001b[0m                                     Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-926cfd9a67a6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mwhile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcap\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misOpened\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mret\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mframe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcap\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mgray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcvtColor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCOLOR_BGR2GRAY\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'frame'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwaitKey\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;36m0xff\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'q'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31merror\u001b[0m: OpenCV(4.1.2) /io/opencv/modules/imgproc/src/color.cpp:182: error: (-215:Assertion failed) !_src.empty() in function 'cvtColor'\n"
     ]
    }
   ],
   "source": [
    "cap = cv2.VideoCapture(\"./videos/sample1.mp4\")\n",
    "while(cap.isOpened()):\n",
    "    ret, frame = cap.read()\n",
    "    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "    cv2.imshow('frame', gray)\n",
    "    if cv2.waitKey(1) & 0xff == ord('q'):\n",
    "        break\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}