Day40_tiny_yolov3_keras_HW.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3031,
     "status": "ok",
     "timestamp": 1575216339249,
     "user": {
      "displayName": "杜靖愷",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCXX8JZZ0DBZWt2P1EC5LvipxZDIWxKz2uCJ7bzpQ=s64",
      "userId": "03195633593178984431"
     },
     "user_tz": -480
    },
    "id": "NCEP-DG0VxlV",
    "outputId": "2d6e6f80-905d-4ff7-c22f-2ded5c0231c2"
   },
   "outputs": [],
   "source": [
    "%tensorflow_version 1.x\n",
    "import tensorflow as tf\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 70
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 3023,
     "status": "ok",
     "timestamp": 1575216339252,
     "user": {
      "displayName": "杜靖愷",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCXX8JZZ0DBZWt2P1EC5LvipxZDIWxKz2uCJ7bzpQ=s64",
      "userId": "03195633593178984431"
     },
     "user_tz": -480
    },
    "id": "vELO-PTVxAtm",
    "outputId": "494921ac-9f94-4453-c62f-8b6f43d04f94"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive \n",
    "drive.mount('/content/gdrive')\n",
    "# 下載基於 keras 的 yolov3 程式碼\n",
    "%cd 'gdrive/My Drive'\n",
    "# !git clone https://github.com/qqwweee/keras-yolo3 # 如果之前已經下載過就可以註解掉\n",
    "%cd keras-yolo3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3012,
     "status": "ok",
     "timestamp": 1575216339253,
     "user": {
      "displayName": "杜靖愷",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCXX8JZZ0DBZWt2P1EC5LvipxZDIWxKz2uCJ7bzpQ=s64",
      "userId": "03195633593178984431"
     },
     "user_tz": -480
    },
    "id": "Avxgh7T7yp2g",
    "outputId": "49a7a631-4514-41da-99da-db57e22204e9"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import subprocess\n",
    "if not os.path.exists(\"model_data/yolo.h5\"):\n",
    "  # 下載 yolov3 的網路權重，並且把權重轉換為 keras 能夠讀取的格式\n",
    "  print(\"Model doesn't exist, downloading...\")\n",
    "  os.system(\"wget https://pjreddie.com/media/files/yolov3.weights\")\n",
    "  print(\"Converting yolov3.weights to yolo.h5...\")\n",
    "  os.system(\"python convert.py yolov3.cfg yolov3.weights model_data/yolo.h5\")\n",
    "else:\n",
    "  print(\"Model exist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "MWPFl6dU1rjr"
   },
   "outputs": [],
   "source": [
    "# 下載圖片範例，如果已經下載過就可以註解掉\n",
    "# !wget https://github.com/pjreddie/darknet/blob/master/data/dog.jpg?raw=true -O dog.jpg\n",
    "# !wget https://github.com/pjreddie/darknet/blob/master/data/horses.jpg?raw=true -O horses.jpg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "-vhfpWt92WZS"
   },
   "outputs": [],
   "source": [
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 593
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5646,
     "status": "ok",
     "timestamp": 1575216341914,
     "user": {
      "displayName": "杜靖愷",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCXX8JZZ0DBZWt2P1EC5LvipxZDIWxKz2uCJ7bzpQ=s64",
      "userId": "03195633593178984431"
     },
     "user_tz": -480
    },
    "id": "jlodol0J42l0",
    "outputId": "5d4f125f-2b32-4a22-fede-e090a4aed01e"
   },
   "outputs": [],
   "source": [
    "image = Image.open('dog.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5634,
     "status": "ok",
     "timestamp": 1575216341915,
     "user": {
      "displayName": "杜靖愷",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCXX8JZZ0DBZWt2P1EC5LvipxZDIWxKz2uCJ7bzpQ=s64",
      "userId": "03195633593178984431"
     },
     "user_tz": -480
    },
    "id": "sncElkSw4l7c",
    "outputId": "cfd6c372-4572-4847-cfba-0d3a607adb36"
   },
   "outputs": [],
   "source": [
    "# 將 yolo.py 所需要的套件載入\n",
    "import colorsys\n",
    "import os\n",
    "from timeit import default_timer as timer\n",
    "\n",
    "import numpy as np\n",
    "from keras import backend as K\n",
    "from keras.models import load_model\n",
    "from keras.layers import Input\n",
    "from PIL import Image, ImageFont, ImageDraw\n",
    "\n",
    "from yolo3.model import yolo_eval, yolo_body, tiny_yolo_body\n",
    "from yolo3.utils import letterbox_image\n",
    "import os\n",
    "from keras.utils import multi_gpu_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "ND4tYZ2u4-Bx"
   },
   "outputs": [],
   "source": [
    "# 定義 YOLO class\n",
    "class YOLO(object):\n",
    "    _defaults = {\n",
    "        \"model_path\": 'model_data/yolo.h5',\n",
    "        \"anchors_path\": 'model_data/yolo_anchors.txt',\n",
    "        \"classes_path\": 'model_data/coco_classes.txt',\n",
    "        \"score\" : 0.3,\n",
    "        \"iou\" : 0.45,\n",
    "        \"model_image_size\" : (416, 416),\n",
    "        \"gpu_num\" : 1,\n",
    "    }\n",
    "\n",
    "    @classmethod\n",
    "    def get_defaults(cls, n):\n",
    "        if n in cls._defaults:\n",
    "            return cls._defaults[n]\n",
    "        else:\n",
    "            return \"Unrecognized attribute name '\" + n + \"'\"\n",
    "\n",
    "    def __init__(self, **kwargs):\n",
    "        self.__dict__.update(self._defaults) # set up default values\n",
    "        self.__dict__.update(kwargs) # and update with user overrides\n",
    "        self.class_names = self._get_class()\n",
    "        self.anchors = self._get_anchors()\n",
    "        self.sess = K.get_session()\n",
    "        self.boxes, self.scores, self.classes = self.generate()\n",
    "\n",
    "    def _get_class(self):\n",
    "        classes_path = os.path.expanduser(self.classes_path)\n",
    "        with open(classes_path) as f:\n",
    "            class_names = f.readlines()\n",
    "        class_names = [c.strip() for c in class_names]\n",
    "        return class_names\n",
    "\n",
    "    def _get_anchors(self):\n",
    "        anchors_path = os.path.expanduser(self.anchors_path)\n",
    "        with open(anchors_path) as f:\n",
    "            anchors = f.readline()\n",
    "        anchors = [float(x) for x in anchors.split(',')]\n",
    "        return np.array(anchors).reshape(-1, 2)\n",
    "\n",
    "    def generate(self):\n",
    "        model_path = os.path.expanduser(self.model_path)\n",
    "        assert model_path.endswith('.h5'), 'Keras model or weights must be a .h5 file.'\n",
    "\n",
    "        # Load model, or construct model and load weights.\n",
    "        num_anchors = len(self.anchors)\n",
    "        num_classes = len(self.class_names)\n",
    "        is_tiny_version = num_anchors==6 # default setting\n",
    "        try:\n",
    "            self.yolo_model = load_model(model_path, compile=False)\n",
    "        except:\n",
    "            self.yolo_model = tiny_yolo_body(Input(shape=(None,None,3)), num_anchors//2, num_classes) \\\n",
    "                if is_tiny_version else yolo_body(Input(shape=(None,None,3)), num_anchors//3, num_classes)\n",
    "            self.yolo_model.load_weights(self.model_path) # make sure model, anchors and classes match\n",
    "        else:\n",
    "            assert self.yolo_model.layers[-1].output_shape[-1] == \\\n",
    "                num_anchors/len(self.yolo_model.output) * (num_classes + 5), \\\n",
    "                'Mismatch between model and given anchor and class sizes'\n",
    "\n",
    "        print('{} model, anchors, and classes loaded.'.format(model_path))\n",
    "\n",
    "        # Generate colors for drawing bounding boxes.\n",
    "        hsv_tuples = [(x / len(self.class_names), 1., 1.)\n",
    "                      for x in range(len(self.class_names))]\n",
    "        self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))\n",
    "        self.colors = list(\n",
    "            map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),\n",
    "                self.colors))\n",
    "        np.random.seed(10101)  # Fixed seed for consistent colors across runs.\n",
    "        np.random.shuffle(self.colors)  # Shuffle colors to decorrelate adjacent classes.\n",
    "        np.random.seed(None)  # Reset seed to default.\n",
    "\n",
    "        # Generate output tensor targets for filtered bounding boxes.\n",
    "        self.input_image_shape = K.placeholder(shape=(2, ))\n",
    "        if self.gpu_num >= 2:\n",
    "            self.yolo_model = multi_gpu_model(self.yolo_model, gpus=self.gpu_num)\n",
    "        boxes, scores, classes = yolo_eval(self.yolo_model.output, self.anchors,\n",
    "                len(self.class_names), self.input_image_shape,\n",
    "                score_threshold=self.score, iou_threshold=self.iou)\n",
    "        return boxes, scores, classes\n",
    "    \n",
    "    # 更改 detect_image 使得其回傳需要的信息\n",
    "    def detect_image(self, image):\n",
    "        start = timer()\n",
    "\n",
    "        if self.model_image_size != (None, None):\n",
    "            assert self.model_image_size[0]%32 == 0, 'Multiples of 32 required'\n",
    "            assert self.model_image_size[1]%32 == 0, 'Multiples of 32 required'\n",
    "            boxed_image = letterbox_image(image, tuple(reversed(self.model_image_size)))\n",
    "        else:\n",
    "            new_image_size = (image.width - (image.width % 32),\n",
    "                              image.height - (image.height % 32))\n",
    "            boxed_image = letterbox_image(image, new_image_size)\n",
    "        image_data = np.array(boxed_image, dtype='float32')\n",
    "\n",
    "        print(image_data.shape)\n",
    "        image_data /= 255.\n",
    "        image_data = np.expand_dims(image_data, 0)  # Add batch dimension.\n",
    "\n",
    "        out_boxes, out_scores, out_classes = self.sess.run(\n",
    "            [self.boxes, self.scores, self.classes],\n",
    "            feed_dict={\n",
    "                self.yolo_model.input: image_data,\n",
    "                self.input_image_shape: [image.size[1], image.size[0]],\n",
    "                K.learning_phase(): 0\n",
    "            })\n",
    "\n",
    "        print('Found {} boxes for {}'.format(len(out_boxes), 'img'))\n",
    "\n",
    "        font = ImageFont.truetype(font='font/FiraMono-Medium.otf',\n",
    "                    size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32'))\n",
    "        thickness = (image.size[0] + image.size[1]) // 300\n",
    "\n",
    "        for i, c in reversed(list(enumerate(out_classes))):\n",
    "            predicted_class = self.class_names[c]\n",
    "            box = out_boxes[i]\n",
    "            score = out_scores[i]\n",
    "\n",
    "            label = '{} {:.2f}'.format(predicted_class, score)\n",
    "            draw = ImageDraw.Draw(image)\n",
    "            label_size = draw.textsize(label, font)\n",
    "\n",
    "            top, left, bottom, right = box\n",
    "            top = max(0, np.floor(top + 0.5).astype('int32'))\n",
    "            left = max(0, np.floor(left + 0.5).astype('int32'))\n",
    "            bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))\n",
    "            right = min(image.size[0], np.floor(right + 0.5).astype('int32'))\n",
    "            print(label, (left, top), (right, bottom))\n",
    "\n",
    "            if top - label_size[1] >= 0:\n",
    "                text_origin = np.array([left, top - label_size[1]])\n",
    "            else:\n",
    "                text_origin = np.array([left, top + 1])\n",
    "\n",
    "            # My kingdom for a good redistributable image drawing library.\n",
    "            for i in range(thickness):\n",
    "                draw.rectangle(\n",
    "                    [left + i, top + i, right - i, bottom - i],\n",
    "                    outline=self.colors[c])\n",
    "            draw.rectangle(\n",
    "                [tuple(text_origin), tuple(text_origin + label_size)],\n",
    "                fill=self.colors[c])\n",
    "            draw.text(text_origin, label, fill=(0, 0, 0), font=font)\n",
    "            del draw\n",
    "\n",
    "        end = timer()\n",
    "        print(end - start)\n",
    "        # return out_boxes, out_scores, out_classes\n",
    "        return image\n",
    "\n",
    "    def close_session(self):\n",
    "        self.sess.close()\n",
    "\n",
    "    # 單純 inference 的 method\n",
    "    def pure_detect_image(self, image):\n",
    "        if self.model_image_size != (None, None):\n",
    "            assert self.model_image_size[0]%32 == 0, 'Multiples of 32 required'\n",
    "            assert self.model_image_size[1]%32 == 0, 'Multiples of 32 required'\n",
    "            boxed_image = letterbox_image(image, tuple(reversed(self.model_image_size)))\n",
    "        else:\n",
    "            new_image_size = (image.width - (image.width % 32),\n",
    "                              image.height - (image.height % 32))\n",
    "            boxed_image = letterbox_image(image, new_image_size)\n",
    "        image_data = np.array(boxed_image, dtype='float32')\n",
    "\n",
    "        # print(image_data.shape)\n",
    "        image_data /= 255.\n",
    "        image_data = np.expand_dims(image_data, 0)  # Add batch dimension.\n",
    "\n",
    "        out_boxes, out_scores, out_classes = self.sess.run(\n",
    "            [self.boxes, self.scores, self.classes],\n",
    "            feed_dict={\n",
    "                self.yolo_model.input: image_data,\n",
    "                self.input_image_shape: [image.size[1], image.size[0]],\n",
    "                K.learning_phase(): 0\n",
    "            })\n",
    "\n",
    "        return out_boxes, out_scores, out_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 530
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 20601,
     "status": "ok",
     "timestamp": 1575216356901,
     "user": {
      "displayName": "杜靖愷",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCXX8JZZ0DBZWt2P1EC5LvipxZDIWxKz2uCJ7bzpQ=s64",
      "userId": "03195633593178984431"
     },
     "user_tz": -480
    },
    "id": "4qCps7TEgtXV",
    "outputId": "afa5f52e-530d-4cfc-82c1-4c7beb06a3c5"
   },
   "outputs": [],
   "source": [
    "yolo = YOLO()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "wd7JfCiOgFcW"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 24553,
     "status": "ok",
     "timestamp": 1575216360869,
     "user": {
      "displayName": "杜靖愷",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCXX8JZZ0DBZWt2P1EC5LvipxZDIWxKz2uCJ7bzpQ=s64",
      "userId": "03195633593178984431"
     },
     "user_tz": -480
    },
    "id": "Ip38sDHn7T3b",
    "outputId": "099ff7d9-c13c-4100-f82a-2e34747d87d2"
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "results = yolo.pure_detect_image(image)\n",
    "end = time.time()\n",
    "print(\"single inference took: %.3fs\" % (end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 24547,
     "status": "ok",
     "timestamp": 1575216360872,
     "user": {
      "displayName": "杜靖愷",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCXX8JZZ0DBZWt2P1EC5LvipxZDIWxKz2uCJ7bzpQ=s64",
      "userId": "03195633593178984431"
     },
     "user_tz": -480
    },
    "id": "xtWGcpIg9H9a",
    "outputId": "3da7b7e7-4e87-4bad-fb7f-98f8e5ee8683"
   },
   "outputs": [],
   "source": [
    "fps = 1 / (end - start)\n",
    "print(\"fps: %.3f\" % (fps))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "pVKeBs0T4Ob0"
   },
   "outputs": [],
   "source": [
    "time_eplased = []\n",
    "for i in range(100):\n",
    "  start = time.time()\n",
    "  results = yolo.pure_detect_image(image)\n",
    "  end = time.time()\n",
    "  time_eplased.append(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 35805,
     "status": "ok",
     "timestamp": 1575216372147,
     "user": {
      "displayName": "杜靖愷",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCXX8JZZ0DBZWt2P1EC5LvipxZDIWxKz2uCJ7bzpQ=s64",
      "userId": "03195633593178984431"
     },
     "user_tz": -480
    },
    "id": "fyy-Xk24iMEg",
    "outputId": "fb7e357e-d6fc-4e00-993b-f2ffe424b499"
   },
   "outputs": [],
   "source": [
    "avg_fps = 1 / np.mean(time_eplased)\n",
    "print(\"avg fps: %.3f\" % avg_fps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": "'nvidia-smi' ���O�����Υ~���R�O�B�i���檺�{���Χ妸�ɡC\n"
    }
   ],
   "source": [
    "!nvidia-smi # 檢測使用的 GPU 型號"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Day40_tiny_yolov3_keras_HW.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}