Day35_yolo_loss.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.1"},"colab":{"name":"Day35_yolo_loss.ipynb","provenance":[],"collapsed_sections":[]}},"cells":[{"cell_type":"markdown","execution_count":null,"metadata":{"colab_type":"text","id":"piG8kRMgbFTz"},"outputs":[],"source":["YOLO 的網路輸出是一個 7x7x30 的 tensor，\n","\n","下面將藉由程式碼實現損失函數的計算，透過損失函數衡量經由 YOLO 模型辨識後的結果和標記檔的差距有多遠。\n","今天的範例，採用的是 YOLO V1 tiny 的網絡\n","\n","資料來源: https://github.com/solaris33/dl_cv_tensorflow_10weeks/blob/master/week10/tensorflow-yolo/yolo/net/yolo_net.py"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{},"colab_type":"code","id":"W7yCo2N5b3D5"},"outputs":[],"source":["# 指定 Google Drive 雲端硬碟的根目錄 drive\n","from google.colab import drive\n","drive.mount('/content/drive')\n","\n","import os\n","import sys\n","import subprocess\n","\n","# 此處為 Google drive 中的文件路徑\n","path = \"/content/drive/.../\n","os.chdir(path)\n","\n","!ls\n","\n","# We'll need to update our path to import from Drive.\n","sys.path.append(os.path.join(path, \"yolo/net\"))\n","\n","if not os.path.exists(\"models/pretrain/yolo_tiny.ckpt\"):\n","  # 下載 YOLO Tiny 的網路權重\n","  print(\"Model doesn't exist, downloading...\")\n","  os.system(\"wget https://drive.google.com/file/d/0B-yiAeTLLamRekxqVE01Yi1RRlk/view?usp=sharing\")\n","\n","else:\n","  print(\"Model exist\")\n","\n","# Now we can import the library and use the function.\n","from yolo_tiny_net import YoloTinyNet\n","\n","# 確保 colob 中使用的 tensorflow 是 1.x 版本而不是 tensorflow 2\n","%tensorflow_version 1.x\n","\n","import tensorflow as tf \n","import cv2\n","import numpy as np\n","import matplotlib.pyplot as plt\n","import keras"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{},"colab_type":"code","id":"7Z5rqCiCbFT4"},"outputs":[],"source":["classes_name =  [\"aeroplane\", \"bicycle\", \"bird\", \"boat\", \"bottle\", \"bus\", \"car\", \"cat\", \"chair\", \"cow\", \"diningtable\", \"dog\", \"horse\", \"motorbike\", \"person\", \"pottedplant\", \"sheep\", \"sofa\", \"train\",\"tvmonitor\"]\n","\n","# 避免執行環境儲存執行結束的變數\n","tf.reset_default_graph()"]},{"cell_type":"markdown","execution_count":null,"metadata":{"colab_type":"text","id":"APeoGwejbFT_"},"outputs":[],"source":["讀入資料集"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":406},"colab_type":"code","executionInfo":{"elapsed":8440,"status":"ok","timestamp":1578970924621,"user":{"displayName":"Mora chen","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mB40f7sDArbZ5_DYq02nNcnLD0Ryaf7AhsASSQeLQ=s64","userId":"03171203089166907199"},"user_tz":-480},"id":"-quWCGQlbFUA","outputId":"16256fc1-f4b2-4fbb-e065-0d65e285baf1"},"outputs":[],"source":["# 下載圖片範例\n","# 如果已經下載過，可以註解掉\n","!wget https://github.com/pjreddie/darknet/blob/master/data/dog.jpg?raw=true -O dog.jpg\n","img = cv2.imread(\"image/dog.jpg\")\n","\n","print(img.shape)\n","h, w, _ = img.shape\n","\n","def show(img):\n","    # 因預設載入的圖片為 BGR，因此現在需要將其轉為 RGB\n","    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) \n","    plt.show()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":269},"colab_type":"code","executionInfo":{"elapsed":8430,"status":"ok","timestamp":1578970924622,"user":{"displayName":"Mora chen","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mB40f7sDArbZ5_DYq02nNcnLD0Ryaf7AhsASSQeLQ=s64","userId":"03171203089166907199"},"user_tz":-480},"id":"VQ_hvwU3bFUE","outputId":"7dd97453-44d4-4fcc-d4ae-751e8b973284"},"outputs":[],"source":["boxes = np.array([\n","            [128, 224, 314, 537],\n","            [475, 85, 689, 170],\n","            [162, 119, 565, 441]\n","        ]).astype(float)\n","\n","# 將 Bounding Boxes 坐標以原圖的 Resolution Normalize 至 0 ~ 1 之間\n","boxes[:, [0, 2]] = boxes[:, [0, 2]] / img.shape[1]\n","boxes[:, [1, 3]] = boxes[:, [1, 3]] / img.shape[0]\n","\n","img_show = img.copy()\n","for x1, y1, x2, y2 in boxes:\n","    cv2.rectangle(img_show, (int(x1 * w), int(y1 * h)), (int(x2 * w), int(y2 * h)), (0, 255, 0), 2)\n","\n","show(img_show)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":321},"colab_type":"code","executionInfo":{"elapsed":9280,"status":"ok","timestamp":1578970925487,"user":{"displayName":"Mora chen","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mB40f7sDArbZ5_DYq02nNcnLD0Ryaf7AhsASSQeLQ=s64","userId":"03171203089166907199"},"user_tz":-480},"id":"IdeZ69vachDL","outputId":"e543a709-7e54-4415-f320-1a255eec3d43"},"outputs":[],"source":["# 模擬影像的標記檔結果\n","# 由於 YOLO 的輸入是 448 * 448，因此圖片的大小需要先設為 448 * 448，所以原本的標記框大小也要對應轉換\n","dy = 448 / 576\n","dx = 448 / 768\n","boxes = np.array([\n","            [128 * dx, 224 * dy, 314 * dx, 537 * dy],\n","            [475 * dx, 85 * dy, 689 * dx, 170 * dy],\n","            [162 * dx, 119 * dy, 565 * dx, 441 * dy]\n","        ]).astype(float)\n","\n","resized_img = cv2.resize(img, (448, 448))\n","img_show = resized_img.copy()\n","for x1, y1, x2, y2 in boxes:\n","    cv2.rectangle(img_show, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 2)\n","\n","show(img_show)\n","\n","# 確認轉換後的標記結果能框住物體的全部\n","print(boxes[0, ])\n","print(boxes[1, ])\n","print(boxes[2, ])"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":70},"colab_type":"code","executionInfo":{"elapsed":9267,"status":"ok","timestamp":1578970925489,"user":{"displayName":"Mora chen","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mB40f7sDArbZ5_DYq02nNcnLD0Ryaf7AhsASSQeLQ=s64","userId":"03171203089166907199"},"user_tz":-480},"id":"qwkNKZiwfF0Q","outputId":"a88ca3a9-183c-421a-876f-472bc19d1d56"},"outputs":[],"source":["# 標記檔：labels 轉成 loss 函數所需的格式：\n","# (xmin, ymin, xmax, ymax) --> (x_center, y_center, w, h, class) in (448, 448)\n","labels = np.zeros((3, 5)) \n","\n","labels [:, 0] = (boxes[:, 0] + boxes[:, 2]) / 2\n","labels [:, 1] = (boxes[:, 1] + boxes[:, 3]) / 2\n","\n","labels [:, 2] = boxes[:, 2] - boxes[:, 0]\n","labels [:, 3] = boxes[:, 3] - boxes[:, 1]\n","\n","labels [0, 4] = 6\n","labels [1, 4] = 1\n","labels [2, 4] = 11\n","\n","print(labels)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":87},"colab_type":"code","executionInfo":{"elapsed":9233,"status":"ok","timestamp":1578970925490,"user":{"displayName":"Mora chen","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mB40f7sDArbZ5_DYq02nNcnLD0Ryaf7AhsASSQeLQ=s64","userId":"03171203089166907199"},"user_tz":-480},"id":"l13_M-_PftRK","outputId":"4e38e784-c724-4d42-88ed-60a89d2a16d1"},"outputs":[],"source":["labels = tf.reshape(labels, (3, 5))\n","labels = tf.cast(labels, tf.float32) \n","\n","print(labels)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{},"colab_type":"code","id":"VI3E-EEibFUH"},"outputs":[],"source":["# 解析模型的輸出\n","def process_predicts(predicts):\n","  # 類別\n","  p_classes = predicts[0, :, :, 0:20]\n","  \n","  # 個別 Bounding Box 所包含物件的機率\n","  c = predicts[0, :, :, 20:22]\n","\n","  # 個別 Bounding Box 的標記框位置\n","  coordinate = predicts[0, :, :, 22:]\n","\n","  p_classes = np.reshape(p_classes, (7, 7, 1, 20))\n","  c = np.reshape(c, (7, 7, 2, 1))\n","\n","  # 對應相乘，產生 𝑝(𝐶𝑙𝑎𝑠𝑠_𝑗 | 𝑜𝑏𝑗𝑒𝑐𝑡) * 𝑃(𝑜𝑏𝑗𝑒𝑐𝑡)\n","  p = c * p_classes\n","\n","  # 返還最大值索引（486）\n","  index = np.argmax(p) \n","  index = np.unravel_index(index, p.shape)\n","\n","  class_num = index[3]\n","\n","  coordinate = np.reshape(coordinate, (7, 7, 2, 4))\n","\n","  max_coordinate = coordinate[index[0], index[1], index[2], :]\n","\n","  x_center = max_coordinate[0]\n","  y_center = max_coordinate[1]\n","  w = max_coordinate[2]\n","  h = max_coordinate[3]\n","  \n","  #index[1] : x 網格位置\n","  x_center = (index[1] + x_center) * (448/7.0)\n","  \n","  # index[0]: y網格位置\n","  y_center = (index[0] + y_center) * (448/7.0)\n","\n","  w *= 448\n","  h *= 448\n","\n","  x_min = x_center - w / 2\n","  y_min = y_center - h / 2\n","\n","  x_max = x_min + w\n","  y_max = y_min + h\n","\n","  return x_min, y_min, x_max, y_max, class_num"]},{"cell_type":"markdown","execution_count":null,"metadata":{"colab_type":"text","id":"xIZxrGwUbFUM"},"outputs":[],"source":["設定 YoloTinyNet，並產生一個預測結果"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{},"colab_type":"code","id":"_9xIMzgFbFUN"},"outputs":[],"source":["common_params = {'image_size': 448, 'num_classes': 20, 'batch_size': 1}\n","net_params = {'cell_size': 7, 'boxes_per_cell': 2, 'weight_decay': 0.0005, 'class_scale': 2, 'object_scale': 1, 'noobject_scale': 0.5, 'coord_scale': 5}\n","\n","net = YoloTinyNet(common_params, net_params, test=False)\n","\n","# 傳值的工作交給 sess.run(), 需要傳入的值放在 feed_dict，並對應每一個 input，而 placeholder 和 feed_dict 是綁定在一起的\n","image = tf.placeholder(tf.float32, (1, 448, 448, 3))\n","predicts = net.inference(image)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"colab_type":"code","executionInfo":{"elapsed":9587,"status":"ok","timestamp":1578970925927,"user":{"displayName":"Mora chen","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mB40f7sDArbZ5_DYq02nNcnLD0Ryaf7AhsASSQeLQ=s64","userId":"03171203089166907199"},"user_tz":-480},"id":"p2RUlgT0bFUS","outputId":"21fa98d9-36da-4540-c2dd-984403c8aebc","scrolled":true},"outputs":[],"source":["# Session 是 Tensorflow 所開啟的一個對話，並執行輸出結果\n","sess = tf.Session()\n","\n","img = cv2.cvtColor(resized_img, cv2.COLOR_BGR2RGB)\n","img = img.astype(np.float32)\n","\n","img = img / 255 * 2 - 1\n","img = np.reshape(img, (1, 448, 448, 3))\n","\n","saver = tf.train.Saver(net.trainable_collection)\n","\n","# 輸入網絡架構的參數檔\n","saver.restore(sess, 'models/pretrain/yolo_tiny.ckpt')\n","\n","# 開啟對話，進行預測\n","predict = sess.run(predicts, feed_dict={image: img})\n","\n","print(predict.shape)\n","print(predict)"]},{"cell_type":"markdown","execution_count":null,"metadata":{"colab_type":"text","id":"BYiq-kKEbFUX"},"outputs":[],"source":["預測出來產生的結果檔"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":158},"colab_type":"code","executionInfo":{"elapsed":9564,"status":"ok","timestamp":1578970925928,"user":{"displayName":"Mora chen","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mB40f7sDArbZ5_DYq02nNcnLD0Ryaf7AhsASSQeLQ=s64","userId":"03171203089166907199"},"user_tz":-480},"id":"gGzfeXuobFUY","outputId":"4c6d949d-31ea-4659-9297-1ebde6fe5ba6"},"outputs":[],"source":["xmin, ymin, xmax, ymax, class_num = process_predicts(np_predict)\n","\n","print(\"output：\")\n","print(xmin)\n","print(ymin)\n","print(xmax)\n","print(ymax)\n","print(class_num)\n","\n","class_name = classes_name[class_num]\n","print(class_name)\n","\n","# 畫出對應框\n","cv2.rectangle(resized_img, (int(xmin), int(ymin)), (int(xmax), int(ymax)), (0, 0, 255))\n","cv2.putText(resized_img, class_name, (int(xmin), int(ymin)), 2, 1.5, (0, 0, 255))\n","\n","# 結果輸出\n","cv2.imwrite('dog_out.jpg', resized_img)"]},{"cell_type":"markdown","execution_count":null,"metadata":{"colab_type":"text","id":"wUS-ZPoNbVmj"},"outputs":[],"source":["\n","\n","接下來我們透過三個函數來計算一張圖片經由 YOLO 模型辨識後的結果和標記檔的差距有多遠。\n","\n","1. iou --> 計算兩個 Bounding Box 的 IoU 值\n","2. body1 --> 轉換過後的標記框，然後和預測框計算損失函數\n","3. loss --> 計算每一個標記框和所有預測框的損失"]},{"cell_type":"markdown","execution_count":null,"metadata":{"colab_type":"text","id":"5ub7qJfrbFVI"},"outputs":[],"source":["loss function 函數"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{},"colab_type":"code","id":"xZhCbltPbFVJ"},"outputs":[],"source":["# 參數的設定\n","image_size = 448\n","num_classes = 20\n","batch_size = 1\n","cell_size = 7\n","boxes_per_cell = 2\n","weight_decay = 0.0005\n","class_scale = 2.0\n","object_scale = 1.0\n","noobject_scale = 0.5\n","coord_scale = 5.0"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{},"colab_type":"code","id":"uQKu4XDtbFVN"},"outputs":[],"source":["def iou(boxes1, boxes2):\n","    # Calculate IoU\n","    # Arguments:\n","    #   boxes1：（代表預測框）4-D tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL, 4]\n","    #   boxes2：（代表實際框）1-D tensor ===> (x_center, y_center, w, h)\n","    \n","    # Return:\n","    #   回傳 IoU 的值\n","    #   IoU: 3-D tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]\n","\n","    # ((x_center - w / 2), (y_center - h / 2), (x_center + w / 2), (y_center + h / 2)) = (左上方座標，右下方座標)\n","    boxes1 = tf.stack([boxes1[:, :, :, 0] - boxes1[:, :, :, 2] / 2, boxes1[:, :, :, 1] - boxes1[:, :, :, 3] / 2,\n","                      boxes1[:, :, :, 0] + boxes1[:, :, :, 2] / 2, boxes1[:, :, :, 1] + boxes1[:, :, :, 3] / 2])\n","    boxes1 = tf.transpose(boxes1, [1, 2, 3, 0])\n","    boxes2 =  tf.stack([boxes2[0] - boxes2[2] / 2, boxes2[1] - boxes2[3] / 2, boxes2[0] + boxes2[2] / 2, boxes2[1] + boxes2[3] / 2])\n","\n","    # Calculate the left up point\n","    # 計算交集的左上方點，和右下方點\n","    left_up = tf.maximum(boxes1[:, :, :, 0:2], boxes2[0:2])\n","    right_down = tf.minimum(boxes1[:, :, :, 2:], boxes2[2:])\n","\n","    # 交集（intersection）\n","    intersection = right_down - left_up\n","    \n","    # 交集面積\n","    inter_square = intersection[:, :, :, 0] * intersection[:, :, :, 1]\n","    \n","    # 只取長寬 > 0 的做計算\n","    mask = tf.cast(intersection[:, :, :, 0] > 0, tf.float32) * tf.cast(intersection[:, :, :, 1] > 0, tf.float32)\n","    \n","    inter_square = mask * inter_square\n","    \n","    # Calculate the boxes1 square and boxes2 square\n","    # 計算聯集面積，等於兩個方形區塊 - 交集面積（calculate the boxes1 square and boxes2 square）\n","    square1 = (boxes1[:, :, :, 2] - boxes1[:, :, :, 0]) * (boxes1[:, :, :, 3] - boxes1[:, :, :, 1])\n","    square2 = (boxes2[2] - boxes2[0]) * (boxes2[3] - boxes2[1])\n","    return inter_square / (square1 + square2 - inter_square + 1e-6)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{},"colab_type":"code","id":"U862awLqbFVT"},"outputs":[],"source":["def cond1(num, object_num, loss, predict, label, nilboy):\n","    return num < object_num"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{},"colab_type":"code","id":"3XQ0icyLbFVb"},"outputs":[],"source":["def body1(num, object_num, loss, predict, labels, nilboy):\n","    # calculate loss\n","    # Args:\n","    #   predict（預測框結果）：3-D tensor [cell_size, cell_size, 5 * boxes_per_cell]\n","    #   labels（標記框） : [max_objects, 5], 5：(x_center, y_center, w, h, class)\n","    #   max_objects, 紀錄物體的總數量\n","\n","    # 用 num 控制現在是計算第幾個物件的標記框，取出來計算\n","\n","    label = labels[num:num + 1, :]\n","    label = tf.reshape(label, [-1])\n","\n","    # Calculate objects tensor [CELL_SIZE, CELL_SIZE]\n","    # 判斷是否有某一物體的標記框中心落在網格 i 中 ==> 1^{object}_{i}\n","    # Objects\n","    min_x = (label[0] - label[2] / 2) / (image_size / cell_size)\n","    max_x = (label[0] + label[2] / 2) / (image_size / cell_size)\n","\n","    min_y = (label[1] - label[3] / 2) / (image_size / cell_size)\n","    max_y = (label[1] + label[3] / 2) / (image_size / cell_size)\n","    \n","    # 計算不大於 min_x 的最大整數\n","    min_x = tf.floor(min_x)\n","    min_y = tf.floor(min_y)\n","    \n","    # 計算不小於 min_x 的最小整數\n","    max_x = tf.ceil(max_x)\n","    max_y = tf.ceil(max_y)\n","\n","    # 相減，計算物體涵蓋的 cell 範圍\n","    temp = tf.cast(tf.stack([max_y - min_y, max_x - min_x]), dtype=tf.int32)\n","    objects = tf.ones(temp, tf.float32)\n","\n","    temp = tf.cast(tf.stack([min_y, cell_size - max_y, min_x, cell_size - max_x]), tf.int32)\n","    temp = tf.reshape(temp, (2, 2))\n","    objects = tf.pad(objects, temp, \"CONSTANT\")\n","\n","    # Calculate objects  tensor [CELL_SIZE, CELL_SIZE]\n","    # Calculate responsible tensor [CELL_SIZE, CELL_SIZE] --> 1^{object}_{ij}\n","    # 判斷第 i 個網格中第 j 個bbox是否負責這個物體\n","    center_x = label[0] / (image_size / cell_size)\n","    center_x = tf.floor(center_x)\n","\n","    center_y = label[1] / (image_size / cell_size)\n","    center_y = tf.floor(center_y)\n","\n","    response = tf.ones([1, 1], tf.float32)\n","\n","    temp = tf.cast(tf.stack([center_y, cell_size - center_y - 1, center_x, cell_size -center_x - 1]), tf.int32)\n","    temp = tf.reshape(temp, (2, 2))\n","    response = tf.pad(response, temp, \"CONSTANT\")\n","    \n","    # calculate iou_predict_truth [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]\n","    # 取出 box\n","    predict_boxes = predict[:, :, num_classes + boxes_per_cell:]\n","    \n","    # 重新 reshape\n","    predict_boxes = tf.reshape(predict_boxes, [cell_size, cell_size, boxes_per_cell, 4])\n","\n","    predict_boxes = predict_boxes * [image_size / cell_size, image_size / cell_size, image_size, image_size]\n","\n","    base_boxes = np.zeros([cell_size, cell_size, 4])\n","\n","    for y in range(cell_size):\n","      for x in range(cell_size):\n","        # nilboy\n","        base_boxes[y, x, :] = [image_size / cell_size * x, image_size / cell_size * y, 0, 0]\n","    base_boxes = np.tile(np.resize(base_boxes, [cell_size, cell_size, 1, 4]), [1, 1, boxes_per_cell, 1])\n","\n","    predict_boxes = base_boxes + predict_boxes\n","\n","    iou_predict_truth = iou(predict_boxes, label[0:4])\n","    # calculate C [cell_size, cell_size, boxes_per_cell]\n","    C = iou_predict_truth * tf.reshape(response, [cell_size, cell_size, 1])\n","\n","    # calculate I tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]\n","    # 判斷第 I 個網格中第 j 個bbox是否負責這個物體\n","    I = iou_predict_truth * tf.reshape(response, (cell_size, cell_size, 1))\n","    \n","    max_I = tf.reduce_max(I, 2, keepdims=True)\n","\n","    I = tf.cast((I >= max_I), tf.float32) * tf.reshape(response, (cell_size, cell_size, 1))\n","\n","    # calculate no_I tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]\n","    no_I = tf.ones_like(I, dtype=tf.float32) - I \n","\n","\n","    p_C = predict[:, :, num_classes:num_classes + boxes_per_cell]\n","\n","    # calculate truth x,y,sqrt_w,sqrt_h 0-D\n","    x = label[0]\n","    y = label[1]\n","\n","    sqrt_w = tf.sqrt(tf.abs(label[2]))\n","    sqrt_h = tf.sqrt(tf.abs(label[3]))\n","    # sqrt_w = tf.abs(label[2])\n","    # sqrt_h = tf.abs(label[3])\n","\n","    # calculate predict p_x, p_y, p_sqrt_w, p_sqrt_h 3-D [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]\n","    p_x = predict_boxes[:, :, :, 0]\n","    p_y = predict_boxes[:, :, :, 1]\n","\n","    p_sqrt_w = tf.sqrt(tf.minimum(image_size * 1.0, tf.maximum(0.0, predict_boxes[:, :, :, 2])))\n","    p_sqrt_h = tf.sqrt(tf.minimum(image_size * 1.0, tf.maximum(0.0, predict_boxes[:, :, :, 3])))\n","    \n","    # calculate truth p 1-D tensor [NUM_CLASSES]\n","    P = tf.one_hot(tf.cast(label[4], tf.int32), num_classes, dtype=tf.float32)\n","\n","    # calculate predict p_P 3-D tensor [CELL_SIZE, CELL_SIZE, NUM_CLASSES]\n","    p_P = predict[:, :, 0:num_classes]\n","\n","    # class_loss\n","    class_loss = tf.nn.l2_loss(tf.reshape(objects, (cell_size, cell_size, 1)) * (p_P - P)) * class_scale\n","    # class_loss = tf.nn.l2_loss(tf.reshape(response, (cell_size, cell_size, 1)) * (p_P - P)) * class_scale\n","\n","    # object_loss\n","    object_loss = tf.nn.l2_loss(I * (p_C - C)) * object_scale\n","\n","    # noobject_loss\n","    noobject_loss = tf.nn.l2_loss(no_I * (p_C)) * noobject_scale\n","\n","    # coord_loss\n","    coord_loss = (tf.nn.l2_loss(I * (p_x - x)/(image_size/cell_size)) +\n","                 tf.nn.l2_loss(I * (p_y - y)/(image_size/cell_size)) +\n","                 tf.nn.l2_loss(I * (p_sqrt_w - sqrt_w))/ image_size +\n","                 tf.nn.l2_loss(I * (p_sqrt_h - sqrt_h))/image_size) * coord_scale\n","\n","    nilboy = I\n","\n","    with tf.Session() as sess1:\n","        print(\"第幾個標記框\",sess1.run(num))\n","        #print(sess1.run(num)) \n","        print(sess1.run(label)) \n","        print(\"class_loss =\",sess1.run(class_loss))\n","        print(\"object_loss=\",sess1.run(object_loss))\n","        print(\"noobject_loss=\",sess1.run(noobject_loss))\n","        print(\"coord_loss=\",sess1.run(coord_loss))\n","    return num + 1, object_num, [loss[0] + class_loss, loss[1] + object_loss, loss[2] + noobject_loss, loss[3] + coord_loss], predict, labels, nilboy"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{},"colab_type":"code","id":"WgHRoc3rbFVf"},"outputs":[],"source":["def loss(predicts, labels, objects_num):\n","    # Add Loss to all the trainable variables\n","\n","    # Args:\n","    #   predicts: [batch_size, cell_size, cell_size, 5 * boxes_per_cell] ===> (num_classes, boxes_per_cell, 4 * boxes_per_cell)\n","    #   labels: 3-D tensor of [batch_size, max_objects, 5]\n","    #   objects_num: 1-D tensor [batch_size]\n","\n","    class_loss = tf.constant(0, tf.float32) # Bounding Box 的類別計算損失\n","    object_loss = tf.constant(0, tf.float32) # Bounding Box 的信心度計算損失\n","    noobject_loss = tf.constant(0, tf.float32) # Bounding Box 的信心度計算損失\n","    coord_loss = tf.constant(0, tf.float32) # Bounding Box 的定位計算損失\n","    loss = [0, 0, 0, 0]\n","    \n","    # 每一張圖都各自計算 loss\n","    for i in range(batch_size):\n","      predict = predicts[i, :, :, :]\n","      label = labels[i, :, :]\n","      object_num = objects_num[i]\n","      nilboy = tf.ones([7, 7, 2])\n","      \n","      # 以每一個標記框分別計算每一張圖是否有那一個物件，並進行損失函數的運算\n","      tuple_results = body1(tf.constant(0), object_num, [class_loss, object_loss, noobject_loss, coord_loss], predict, label, nilboy)\n","      for j in range(4):\n","        loss[j] = loss[j] + tuple_results[2][j]\n","      \n","      tuple_results = body1(tf.constant(1), object_num, [class_loss, object_loss, noobject_loss, coord_loss], predict, label, nilboy)\n","      for j in range(4):\n","        loss[j] = loss[j] + tuple_results[2][j]\n","      \n","      tuple_results = body1(tf.constant(2), object_num, [class_loss, object_loss, noobject_loss, coord_loss], predict, label, nilboy)    \n","      for j in range(4):\n","        loss[j] = loss[j] + tuple_results[2][j]\n","      \n","      nilboy = tuple_results[5]\n","    \n","    return loss"]},{"cell_type":"markdown","execution_count":null,"metadata":{"colab_type":"text","id":"ARNv7PLmbFVl"},"outputs":[],"source":["### 計算標記框和預測框的loss function"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":386},"colab_type":"code","executionInfo":{"elapsed":11627,"status":"ok","timestamp":1578970928124,"user":{"displayName":"Mora chen","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mB40f7sDArbZ5_DYq02nNcnLD0Ryaf7AhsASSQeLQ=s64","userId":"03171203089166907199"},"user_tz":-480},"id":"cAUyFiKmbFVn","outputId":"416f5fc1-5522-4002-8ec5-67d301dadb60"},"outputs":[],"source":["predicts = tf.reshape(np_predict,(1, 7, 7, 30))\n","labels = tf.reshape(labels, (1, 3, 5))\n","output_loss = loss(predicts, labels, tf.constant(3, shape=[1]))\n","\n","print(\"預測結果和標記框的損失量\")\n","\n","with tf.Session() as sess: \n","    print(sess.run(output_loss)) \n","\n","sess.close()"]}]}