train_human_pose.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mask R-CNN - Human pose estimation\n",
    "\n",
    "Inspect and visualize data loading and pre-processing code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\programdata\\anaconda3\\envs\\py35\\lib\\site-packages\\h5py\\__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import itertools\n",
    "import math\n",
    "import logging\n",
    "import json\n",
    "import re\n",
    "import random\n",
    "from collections import OrderedDict\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "import matplotlib.lines as lines\n",
    "from matplotlib.patches import Polygon\n",
    "\n",
    "import utils\n",
    "import visualize\n",
    "from visualize import display_images\n",
    "import model as modellib\n",
    "from model import log\n",
    "\n",
    "%matplotlib inline \n",
    "\n",
    "ROOT_DIR = os.getcwd()\n",
    "MODEL_DIR = os.path.join(ROOT_DIR, \"logs\")\n",
    "# Local path to trained weights file\n",
    "COCO_MODEL_PATH = os.path.join(ROOT_DIR, \"mask_rcnn_coco.h5\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configurations\n",
    "\n",
    "Run one of the code blocks below to import and load the configurations to use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MS COCO Dataset\n",
    "import coco\n",
    "config = coco.CocoConfig()\n",
    "COCO_DIR = \"D:/Github/FastMaskRCNN/data/coco\"  # TODO: enter your own path here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=12.85s)\n",
      "creating index...\n",
      "index created!\n",
      "Skeleton: (19, 2)\n",
      "Keypoint names: (17,)\n",
      "loading annotations into memory...\n",
      "Done (t=0.69s)\n",
      "creating index...\n",
      "index created!\n",
      "Skeleton: (19, 2)\n",
      "Keypoint names: (17,)\n",
      "Train Keypoints Image Count: 64115\n",
      "Train Keypoints Class Count: 2\n",
      "  0. BG                                                \n",
      "  1. person                                            \n",
      "Val Keypoints Image Count: 2693\n",
      "Val Keypoints Class Count: 2\n",
      "  0. BG                                                \n",
      "  1. person                                            \n"
     ]
    }
   ],
   "source": [
    "# MS COCO Dataset\n",
    "import coco\n",
    "config = coco.CocoConfig()\n",
    "COCO_DIR = \"D:/Github/FastMaskRCNN/data/coco\"  # TODO: enter value here\n",
    "# Load dataset\n",
    "assert config.NAME == \"coco\"\n",
    "# Training dataset\n",
    "# load person keypoints dataset\n",
    "train_dataset_keypoints = coco.CocoDataset(task_type=\"person_keypoints\")\n",
    "train_dataset_keypoints.load_coco(COCO_DIR, \"train\")\n",
    "train_dataset_keypoints.prepare()\n",
    "\n",
    "#Validation dataset\n",
    "val_dataset_keypoints = coco.CocoDataset(task_type=\"person_keypoints\")\n",
    "val_dataset_keypoints.load_coco(COCO_DIR, \"val\")\n",
    "val_dataset_keypoints.prepare()\n",
    "\n",
    "print(\"Train Keypoints Image Count: {}\".format(len(train_dataset_keypoints.image_ids)))\n",
    "print(\"Train Keypoints Class Count: {}\".format(train_dataset_keypoints.num_classes))\n",
    "for i, info in enumerate(train_dataset_keypoints.class_info):\n",
    "    print(\"{:3}. {:50}\".format(i, info['name']))\n",
    "\n",
    "print(\"Val Keypoints Image Count: {}\".format(len(val_dataset_keypoints.image_ids)))\n",
    "print(\"Val Keypoints Class Count: {}\".format(val_dataset_keypoints.num_classes))\n",
    "for i, info in enumerate(val_dataset_keypoints.class_info):\n",
    "    print(\"{:3}. {:50}\".format(i, info['name']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ceate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create model object in inference mode.\n",
    "model = modellib.MaskRCNN(mode=\"training\", model_dir=MODEL_DIR, config=config)\n",
    "\n",
    "# Load weights trained on MS-COCO\n",
    "model.load_weights(COCO_MODEL_PATH, by_name=True,exclude=[\"mrcnn_class_logits\", \"mrcnn_bbox_fc\", \n",
    "                                \"mrcnn_bbox\", \"mrcnn_mask\"])\n",
    "print(\"Loading weights from \", model_path)\n",
    "# model.keras_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train heads\n",
      "\n",
      "Starting at epoch 0. LR=0.002\n",
      "\n",
      "Checkpoint Path: D:\\Github\\Mask_RCNN\\logs\\coco20180327T1324\\mask_rcnn_coco_{epoch:04d}.h5\n",
      "Selecting layers to train\n",
      "fpn_c5p5               (Conv2D)\n",
      "fpn_c4p4               (Conv2D)\n",
      "fpn_c3p3               (Conv2D)\n",
      "fpn_c2p2               (Conv2D)\n",
      "fpn_p5                 (Conv2D)\n",
      "fpn_p2                 (Conv2D)\n",
      "fpn_p3                 (Conv2D)\n",
      "fpn_p4                 (Conv2D)\n",
      "In model:  rpn_model\n",
      "    rpn_conv_shared        (Conv2D)\n",
      "    rpn_class_raw          (Conv2D)\n",
      "    rpn_bbox_pred          (Conv2D)\n",
      "mrcnn_keypoint_mask_conv1   (TimeDistributed)\n",
      "mrcnn_keypoint_mask_bn1   (TimeDistributed)\n",
      "mrcnn_keypoint_mask_conv2   (TimeDistributed)\n",
      "mrcnn_keypoint_mask_bn2   (TimeDistributed)\n",
      "mrcnn_keypoint_mask_conv3   (TimeDistributed)\n",
      "mrcnn_keypoint_mask_bn3   (TimeDistributed)\n",
      "mrcnn_keypoint_mask_conv4   (TimeDistributed)\n",
      "mrcnn_keypoint_mask_bn4   (TimeDistributed)\n",
      "mrcnn_keypoint_mask_conv5   (TimeDistributed)"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\programdata\\anaconda3\\envs\\py35\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_impl.py:96: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "mrcnn_keypoint_mask_bn5   (TimeDistributed)\n",
      "mrcnn_keypoint_mask_conv6   (TimeDistributed)\n",
      "mrcnn_mask_conv1       (TimeDistributed)\n",
      "mrcnn_keypoint_mask_bn6   (TimeDistributed)\n",
      "mrcnn_mask_bn1         (TimeDistributed)\n",
      "mrcnn_keypoint_mask_conv7   (TimeDistributed)\n",
      "mrcnn_mask_conv2       (TimeDistributed)\n",
      "mrcnn_keypoint_mask_bn7   (TimeDistributed)\n",
      "mrcnn_mask_bn2         (TimeDistributed)\n",
      "mrcnn_class_conv1      (TimeDistributed)\n",
      "mrcnn_class_bn1        (TimeDistributed)\n",
      "mrcnn_keypoint_mask_conv8   (TimeDistributed)\n",
      "mrcnn_mask_conv3       (TimeDistributed)\n",
      "mrcnn_keypoint_mask_bn8   (TimeDistributed)\n",
      "mrcnn_mask_bn3         (TimeDistributed)\n",
      "mrcnn_class_conv2      (TimeDistributed)\n",
      "mrcnn_class_bn2        (TimeDistributed)\n",
      "mrcnn_keypoint_mask_deconv   (TimeDistributed)\n",
      "mrcnn_mask_conv4       (TimeDistributed)\n",
      "mrcnn_mask_bn4         (TimeDistributed)\n",
      "mrcnn_bbox_fc          (TimeDistributed)\n",
      "mrcnn_mask_deconv      (TimeDistributed)\n",
      "mrcnn_class_logits     (TimeDistributed)\n",
      "mrcnn_mask             (TimeDistributed)\n",
      "Epoch 1/15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\programdata\\anaconda3\\envs\\py35\\lib\\site-packages\\scipy\\ndimage\\interpolation.py:616: UserWarning: From scipy 0.13.0, the output shape of zoom() is calculated with round() instead of int() - for these inputs the size of the returned array has changed.\n",
      "  \"the returned array has changed.\", UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  10/1000 [..............................] - ETA: 1:02:50 - loss: 9.7492 - rpn_class_loss: 0.0223 - rpn_bbox_loss: 0.2071 - mrcnn_class_loss: 0.4194 - mrcnn_bbox_loss: 0.9257 - keypoint_mrcnn_mask_loss: 7.2386 - mrcnn_mask_loss: 0.9361"
     ]
    }
   ],
   "source": [
    "# Training - Stage 1\n",
    "print(\"Train heads\")\n",
    "model.train(train_dataset_keypoints, val_dataset_keypoints,\n",
    "            learning_rate=config.LEARNING_RATE,\n",
    "            epochs=15,\n",
    "            layers='heads')\n",
    "# Training - Stage 2\n",
    "# Finetune layers from ResNet stage 4 and up\n",
    "print(\"Training Resnet layer 4+\")\n",
    "model.train(train_dataset_keypoints, val_dataset_keypoints,\n",
    "            learning_rate=config.LEARNING_RATE / 10,\n",
    "            epochs=20,\n",
    "            layers='4+')\n",
    "# Training - Stage 3\n",
    "# Finetune layers from ResNet stage 3 and up\n",
    "print(\"Training Resnet layer 3+\")\n",
    "model.train(train_dataset_keypoints, val_dataset_keypoints,\n",
    "            learning_rate=config.LEARNING_RATE / 100,\n",
    "            epochs=100,\n",
    "            layers='all')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}