Code/YOLO_object_detection/Implementation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Implementation.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uxgk1T8mJlhL",
        "cellView": "both",
        "outputId": "dd2d6677-3f37-4e3d-cc5e-0f295a7ef58a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#@title\n",
        "#changing tensorflow version for compatibility\n",
        "%tensorflow_version 1.x\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)\n",
        "\n",
        "#device configuration\n",
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))\n",
        "\n",
        "# use this command to use gpu\n",
        "# with tf.device('/device:GPU:0'):"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n",
            "1.15.2\n",
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WqJj2PQd0b6V"
      },
      "source": [
        "#draw boxes\n",
        "import numpy as np\n",
        "from PIL import Image, ImageDraw\n",
        "from google.colab import drive\n",
        "\n",
        "\n",
        "def draw_bbox(img, bbox, text):\n",
        "    h, w = img.shape[0], img.shape[1]\n",
        "    x0, y0, x1, y1 = max(bbox[0], 0), max(bbox[1], 0), min(bbox[2], w-1), min(bbox[3], h-1)\n",
        "    drawed_img = img * 1\n",
        "    color = np.zeros([x1 - x0 + 1, 3])\n",
        "    color[:, 1] = np.ones([x1 - x0 + 1]) * 255#Green rectangle\n",
        "    drawed_img[y0, x0:x1 + 1, :] = color\n",
        "    drawed_img[y1, x0:x1 + 1, :] = color\n",
        "    color = np.zeros([y1 - y0 + 1, 3])\n",
        "    color[:, 1] = np.ones([y1 - y0 + 1]) * 255  # Green rectangle\n",
        "    drawed_img[y0:y1 + 1, x0, :] = color\n",
        "    drawed_img[y0:y1 + 1, x1, :] = color\n",
        "    #type text\n",
        "    drawed_img = Image.fromarray(np.uint8(drawed_img))\n",
        "    draw = ImageDraw.Draw(drawed_img)\n",
        "    x = int(bbox[0])\n",
        "    y = int(bbox[1])\n",
        "    draw.text((x, y), text)\n",
        "    drawed_img = np.array(drawed_img)\n",
        "    return drawed_img"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "svsCizIoSR8g",
        "outputId": "1496469c-e4a6-4eb8-a4ba-0ca769b4d48a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i2gmYLoNMYcC",
        "outputId": "d3eaab6b-d871-4def-c5ba-f0c6f14ebfc6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "! wget http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-11-03 17:48:51--  http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar\n",
            "Resolving host.robots.ox.ac.uk (host.robots.ox.ac.uk)... 129.67.94.152\n",
            "Connecting to host.robots.ox.ac.uk (host.robots.ox.ac.uk)|129.67.94.152|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 460032000 (439M) [application/x-tar]\n",
            "Saving to: ‘VOCtrainval_06-Nov-2007.tar’\n",
            "\n",
            "VOCtrainval_06-Nov- 100%[===================>] 438.72M  9.07MB/s    in 53s     \n",
            "\n",
            "2020-11-03 17:49:44 (8.31 MB/s) - ‘VOCtrainval_06-Nov-2007.tar’ saved [460032000/460032000]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e3Rac7NnMXGm"
      },
      "source": [
        "!mkdir /content/VOC\n",
        "!tar -xvf  '/content/VOCtrainval_06-Nov-2007.tar' -C '/content/VOC'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QP2lWgUw43u_"
      },
      "source": [
        "#networks VGG-16\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "#convolutional layer\n",
        "def conv(inputs, w, b, trainable=False):\n",
        "    w = tf.Variable(initial_value=w, trainable=trainable)\n",
        "    b = tf.Variable(initial_value=b, trainable=trainable)\n",
        "    inputs = tf.nn.conv2d(inputs, w, [1, 1, 1, 1], \"SAME\")\n",
        "    return tf.nn.bias_add(inputs, b)\n",
        "#max pooling layer\n",
        "def max_pooling(inputs):\n",
        "    return tf.nn.max_pool(inputs, [1, 2, 2, 1], [1, 2, 2, 1], \"SAME\")\n",
        "#relu\n",
        "def relu(inputs):\n",
        "    return tf.nn.relu(inputs)\n",
        "#fully connected layer\n",
        "def vgg_fully_connected(name, inputs, init_W, init_b):\n",
        "    with tf.variable_scope(name):\n",
        "        inputs = tf.layers.flatten(inputs)\n",
        "        W = tf.Variable(initial_value=init_W)\n",
        "        b = tf.Variable(initial_value=init_b)\n",
        "        inputs = tf.matmul(inputs, W)\n",
        "        inputs = tf.nn.bias_add(inputs, b)\n",
        "    return inputs\n",
        "\n",
        "def conv_(name, inputs, nums_out, k_size, strides, padding=\"SAME\"):\n",
        "    nums_in = inputs.shape[-1]\n",
        "    with tf.variable_scope(name):\n",
        "        W = tf.get_variable(\"W\", [k_size, k_size, nums_in, nums_out], initializer=tf.truncated_normal_initializer(stddev=0.02))\n",
        "        b = tf.get_variable(\"b\", [nums_out], initializer=tf.constant_initializer([0.]))\n",
        "        inputs = tf.nn.conv2d(inputs, W, [1, strides, strides, 1], padding=padding)\n",
        "        inputs = tf.nn.bias_add(inputs, b)\n",
        "    return inputs\n",
        "\n",
        "def fc(name, inputs, nums_out):\n",
        "    nums_in = inputs.shape[-1]\n",
        "    with tf.variable_scope(name):\n",
        "        W = tf.get_variable(\"W\", [nums_in, nums_out], initializer=tf.truncated_normal_initializer(stddev=0.02))\n",
        "        b = tf.get_variable(\"b\", [nums_out], initializer=tf.constant_initializer([0.]))\n",
        "        inputs = tf.matmul(inputs, W)\n",
        "    return tf.nn.bias_add(inputs, b)\n",
        "\n",
        "\n",
        "def vgg16(inputs):\n",
        "    inputs = tf.reverse(inputs, [-1]) - tf.constant([103.939, 116.779, 123.68])\n",
        "    inputs /= 255.0\n",
        "    para = np.load(\"/content/drive/My Drive/vgg16.npy.npy（副本）\", encoding=\"latin1\",allow_pickle=True).item()\n",
        "    inputs = relu(conv(inputs, para[\"conv1_1\"][0], para[\"conv1_1\"][1]))\n",
        "    inputs = relu(conv(inputs, para[\"conv1_2\"][0], para[\"conv1_2\"][1]))\n",
        "    inputs = max_pooling(inputs)\n",
        "    inputs = relu(conv(inputs, para[\"conv2_1\"][0], para[\"conv2_1\"][1]))\n",
        "    inputs = relu(conv(inputs, para[\"conv2_2\"][0], para[\"conv2_2\"][1]))\n",
        "    inputs = max_pooling(inputs)\n",
        "    inputs = relu(conv(inputs, para[\"conv3_1\"][0], para[\"conv3_1\"][1], True))\n",
        "    inputs = relu(conv(inputs, para[\"conv3_2\"][0], para[\"conv3_2\"][1], True))\n",
        "    inputs = relu(conv(inputs, para[\"conv3_3\"][0], para[\"conv3_3\"][1], True))\n",
        "    inputs = max_pooling(inputs)\n",
        "    inputs = relu(conv(inputs, para[\"conv4_1\"][0], para[\"conv4_1\"][1], True))\n",
        "    inputs = relu(conv(inputs, para[\"conv4_2\"][0], para[\"conv4_2\"][1], True))\n",
        "    inputs = relu(conv(inputs, para[\"conv4_3\"][0], para[\"conv4_3\"][1], True))\n",
        "    inputs = max_pooling(inputs)\n",
        "    inputs = relu(conv(inputs, para[\"conv5_1\"][0], para[\"conv5_1\"][1], True))\n",
        "    inputs = relu(conv(inputs, para[\"conv5_2\"][0], para[\"conv5_2\"][1], True))\n",
        "    inputs = relu(conv(inputs, para[\"conv5_3\"][0], para[\"conv5_3\"][1], True))\n",
        "    inputs = max_pooling(inputs)\n",
        "    inputs = tf.layers.flatten(inputs)\n",
        "    inputs = fc(\"full\", inputs, 512)\n",
        "    inputs = fc(\"logits\", inputs, 7*7*30)\n",
        "    inputs = tf.reshape(inputs, [-1, 7, 7, 30])\n",
        "    return tf.sigmoid(inputs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QeKQRYlG3yck",
        "outputId": "97000e2f-61ba-4a68-8349-eb005c711857",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#loss_YOLO\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def yolo_loss(prediction, labels, H=448, W=448, S=7):\n",
        "    #prediction: batchsize x 7 x 7 x 30, bboxes: batchsize x 7 x 7 x (0:8), confidences: batchsize x 7 x 7 x (8:10), class: batchsize x 7 x 7 x (10:30)\n",
        "    #labels: batchsize x 7 x 7 x 25, response_mask: batchsize x 7 x 7 x (0:1), bbox: batchsize x 7 x 7 x (1:5), class: batchsize: batchsize x 7 x 7 x (5:25)\n",
        "    pred_bboxes = prediction[:, :, :, 0:8]\n",
        "    pred_bboxes = tf.reshape(pred_bboxes, [-1, 7, 7, 2, 4])\n",
        "    pred_confidence = prediction[:, :, :, 8:10]\n",
        "    pred_class = prediction[:, :, :, 10:]\n",
        "\n",
        "    #groundtruth\n",
        "    label_mask = labels[:, :, :, 0:1]\n",
        "    label_bboxes = labels[:, :, :, 1:5]\n",
        "    label_bboxes = tf.tile(label_bboxes, multiples=[1, 1, 1, 2])\n",
        "    label_bboxes = tf.reshape(label_bboxes, [-1, 7, 7, 2, 4])\n",
        "    label_class = labels[:, :, :, 5:]\n",
        "\n",
        "    #prediction normalized offset -> [c_x, c_y, h, w]\n",
        "    cell_h = H / S #normalising the height\n",
        "    cell_w = W / S #normalising the width\n",
        "    temp = tf.constant([[0., 1., 2., 3., 4., 5., 6.]])\n",
        "    temp = tf.tile(temp, multiples=[7, 1])\n",
        "    col = temp[tf.newaxis, :, :, tf.newaxis, tf.newaxis]#dimension 7 x 7 -> 1 x 7 x 7 x 1 x 1\n",
        "    row = tf.transpose(col, perm=[0, 2, 1, 3, 4])\n",
        "    pred_bboxes_original = tf.concat([(pred_bboxes[:, :, :, :, 0:1] + col) * cell_w,\n",
        "                                      (pred_bboxes[:, :, :, :, 1:2] + row) * cell_h,\n",
        "                                       tf.square(pred_bboxes[:, :, :, :, 2:3]) * H,\n",
        "                                       tf.square(pred_bboxes[:, :, :, :, 3:4]) * W], axis=-1)\n",
        "    label_bboxes_original = tf.concat([(label_bboxes[:, :, :, :, 0:1] + col) * cell_w,\n",
        "                                      (label_bboxes[:, :, :, :, 1:2] + row) * cell_h,\n",
        "                                      tf.square(label_bboxes[:, :, :, :, 2:3]) * H,\n",
        "                                      tf.square(label_bboxes[:, :, :, :, 3:4]) * W], axis=-1)\n",
        "    iou = cal_iou(pred_bboxes_original, label_bboxes_original)#input: batchsize x 7 x 7 x 2 x 4, output: batchsize x 7 x 7 x 2\n",
        "    max_iou = tf.reduce_max(iou, axis=-1, keep_dims=True)#output: batchsize x 7 x 7 x 1\n",
        "    mask_obj = tf.cast(tf.greater_equal(iou, max_iou), dtype=tf.float32) * label_mask\n",
        "    loss_bboxes = tf.reduce_mean(tf.reduce_sum(tf.square(pred_bboxes - label_bboxes) * mask_obj[:, :, :, :, tf.newaxis], axis=[1, 2, 3, 4]))\n",
        "    loss_confidence_obj = tf.reduce_mean(tf.reduce_sum(tf.square(pred_confidence - 1) * mask_obj, axis=[1, 2, 3]))\n",
        "    loss_confidence_noobj = tf.reduce_mean(tf.reduce_sum(tf.square(pred_confidence) * (1 - mask_obj), axis=[1, 2, 3]))\n",
        "    loss_class = tf.reduce_mean(tf.reduce_sum(tf.square(pred_class - label_class) * label_mask, axis=[1, 2, 3]))\n",
        "    loss = 5 * loss_bboxes + loss_confidence_obj + 0.5 * loss_confidence_noobj + loss_class #lamda values for coord and noobj respectively are 5 and 0.5\n",
        "    return loss, loss_bboxes, loss_confidence_obj, loss_confidence_noobj, loss_class\n",
        "\n",
        "def cal_iou(bboxes1, bboxes2):\n",
        "    #bboxes: [batchsize, 7, 7, 2, 4] with [center_x, center_y, h, w]\n",
        "    cx, cx_ = bboxes1[:, :, :, :, 0], bboxes2[:, :, :, :, 0]\n",
        "    cy, cy_ = bboxes1[:, :, :, :, 1], bboxes2[:, :, :, :, 1]\n",
        "    h, h_ = bboxes1[:, :, :, :, 2], bboxes2[:, :, :, :, 2]\n",
        "    w, w_ = bboxes1[:, :, :, :, 3], bboxes2[:, :, :, :, 3]\n",
        "    x1, x1_ = cx - w / 2, cx_ - w_ / 2\n",
        "    x2, x2_ = cx + w / 2, cx_ + w_ / 2\n",
        "    y1, y1_ = cy - h / 2, cy_ - h_ / 2\n",
        "    y2, y2_ = cy + h / 2, cy_ + h_ / 2\n",
        "    x_inter1 = tf.maximum(x1, x1_)\n",
        "    x_inter2 = tf.minimum(x2, x2_)\n",
        "    y_inter1 = tf.maximum(y1, y1_)\n",
        "    y_inter2 = tf.minimum(y2, y2_)\n",
        "    h_inter = tf.maximum(0., y_inter2 - y_inter1)\n",
        "    w_inter = tf.maximum(0., x_inter2 - x_inter1)\n",
        "    area_inter = h_inter * w_inter\n",
        "    area_union = h * w + h_ * w_ - area_inter\n",
        "    iou = area_inter / area_union\n",
        "    return iou\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    prediction = tf.placeholder(tf.float32, [None, 7, 7, 30])\n",
        "    labels = tf.placeholder(tf.float32, [None, 7, 7, 25])\n",
        "    yolo_loss(prediction, labels)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-7-c220d7cd8446>:37: calling reduce_max_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "keep_dims is deprecated, use keepdims instead\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yEmEDvRC3-d6"
      },
      "source": [
        "import xml.etree.cElementTree as ET\n",
        "import numpy as np\n",
        "import os\n",
        "from PIL import Image\n",
        "import scipy.misc as misc\n",
        "import scipy.io as sio\n",
        "\n",
        "\n",
        "OBJECT_NAMES = [\"tvmonitor\", \"train\", \"sofa\", \"sheep\", \"cat\", \"chair\", \"bottle\", \"motorbike\", \"boat\", \"bird\",\n",
        "                   \"person\", \"aeroplane\", \"dog\", \"pottedplant\", \"cow\", \"bus\", \"diningtable\", \"horse\", \"bicycle\", \"car\"]\n",
        "EPSILON = 1e-8\n",
        "\n",
        "def read_xml(path):\n",
        "    tree = ET.parse(path)\n",
        "    root = tree.getroot()\n",
        "    objects = root.findall(\"object\")\n",
        "    imgname = root.find(\"filename\").text\n",
        "    gt_bbox = np.zeros([objects.__len__(), 4], dtype=np.int32)\n",
        "    name_bbox = []\n",
        "    for i, obj in enumerate(objects):\n",
        "        objectname = obj.find(\"name\").text\n",
        "        bbox = np.zeros([4], dtype=np.int32)\n",
        "        xmin = int(obj.find(\"bndbox\").find(\"xmin\").text)\n",
        "        ymin = int(obj.find(\"bndbox\").find(\"ymin\").text)\n",
        "        xmax = int(obj.find(\"bndbox\").find(\"xmax\").text)\n",
        "        ymax = int(obj.find(\"bndbox\").find(\"ymax\").text)\n",
        "        bbox[0], bbox[1], bbox[2], bbox[3] = xmin, ymin, xmax, ymax\n",
        "        name_bbox.append(objectname)\n",
        "        gt_bbox[i, :] = bbox\n",
        "\n",
        "    return imgname, gt_bbox, name_bbox\n",
        "\n",
        "def cal_iou_1(bbox1, bbox2):\n",
        "    #bbox = [x1, y1, x2, y2]\n",
        "    x1, y1, x1_, y1_ = bbox1[0], bbox1[1], bbox1[2], bbox1[3]\n",
        "    x2, y2, x2_, y2_ = bbox2[0], bbox2[1], bbox2[2], bbox2[3]\n",
        "    x0 = max(x1, x2)\n",
        "    y0 = max(y1, y2)\n",
        "    x0_ = min(x1_, x2_)\n",
        "    y0_ = min(y1_, y2_)\n",
        "\n",
        "    if x0 >= x0_ or y0 >= y0_:\n",
        "        iou = 0\n",
        "    else:\n",
        "        inter_area = (x0_ - x0) * (y0_ - y0)\n",
        "        bbox1_area = (x1_ - x1) * (y1_ - y1)\n",
        "        bbox2_area = (x2_ - x2) * (y2_ - y2)\n",
        "        union_area = bbox1_area + bbox2_area - inter_area\n",
        "\n",
        "        iou = inter_area / union_area\n",
        "    return iou\n",
        "\n",
        "def ToScaleImg(img, tar_h, tar_w, raw_bboxes):\n",
        "    h, w = img.shape[0], img.shape[1]\n",
        "    nums_bbox = raw_bboxes.shape[0]\n",
        "    tar_bboxes = np.zeros_like(raw_bboxes)\n",
        "    for i in range(nums_bbox):\n",
        "        bbox = raw_bboxes[i]\n",
        "        x0, y0, x1, y1 = bbox[0], bbox[1], bbox[2], bbox[3]\n",
        "        x0 = tar_w / w * x0\n",
        "        x1 = tar_w / w * x1\n",
        "        y0 = tar_h / h * y0\n",
        "        y1 = tar_h / h * y1\n",
        "        tar_bboxes[i, 0], tar_bboxes[i, 1] = x0, y0\n",
        "        tar_bboxes[i, 2], tar_bboxes[i, 3] = x1, y1\n",
        "    scaled_img = misc.imresize(img, [tar_h, tar_w])\n",
        "    return scaled_img, tar_bboxes\n",
        "\n",
        "\n",
        "def read_batch(img_path, xml_path, batch_size, img_h=448, img_w=448):\n",
        "    xml_lists = os.listdir(xml_path)\n",
        "    nums = xml_lists.__len__()\n",
        "    rand_idx = np.random.randint(0, nums, [batch_size])\n",
        "    batch_bboxes = np.zeros([batch_size, 7, 7, 4])\n",
        "    batch_classes = np.zeros([batch_size, 7, 7, 20])\n",
        "    batch_img = np.zeros([batch_size, img_h, img_w, 3])\n",
        "    cell_h = img_h / 7\n",
        "    cell_w = img_w / 7\n",
        "    for j in range(batch_size):\n",
        "        imgname, gt_bbox, name_bbox = read_xml(xml_path + xml_lists[rand_idx[j]])\n",
        "        img = np.array(Image.open(img_path + imgname))\n",
        "        scaled_img, scaled_bbox = ToScaleImg(img, img_h, img_w, gt_bbox)\n",
        "        batch_img[j, :, :, :] = scaled_img\n",
        "        for i in range(scaled_bbox.shape[0]):\n",
        "            c_x = (scaled_bbox[i, 0] + scaled_bbox[i, 2]) / 2\n",
        "            c_y = (scaled_bbox[i, 1] + scaled_bbox[i, 3]) / 2\n",
        "            h = scaled_bbox[i, 3] - scaled_bbox[i, 1]\n",
        "            w = scaled_bbox[i, 2] - scaled_bbox[i, 0]\n",
        "            col = int(c_x // cell_w)\n",
        "            row = int(c_y // cell_h)\n",
        "            offset_x = c_x / cell_w - col\n",
        "            offset_y = c_y / cell_h - row\n",
        "            offset_h = np.sqrt(h / img_h)\n",
        "            offset_w = np.sqrt(w / img_w)\n",
        "            batch_bboxes[j, row, col, 0], batch_bboxes[j, row, col, 1] = offset_x, offset_y\n",
        "            batch_bboxes[j, row, col, 2], batch_bboxes[j, row, col, 3] = offset_h, offset_w\n",
        "            index = OBJECT_NAMES.index(name_bbox[i])\n",
        "            batch_classes[j, row, col, index] = 1\n",
        "    batch_labels = np.zeros([batch_size, 7, 7, 25])\n",
        "    batch_response = np.sum(batch_classes, axis=-1, keepdims=True)\n",
        "    batch_labels[:, :, :, 0:1] = batch_response\n",
        "    batch_labels[:, :, :, 1:5] = batch_bboxes\n",
        "    batch_labels[:, :, :, 5:] = batch_classes\n",
        "    return batch_img, batch_labels\n",
        "\n",
        "def img2mat(imgpath, xmlpath):\n",
        "    filenames = os.listdir(xmlpath)\n",
        "    nums = filenames.__len__()\n",
        "    imgs = np.zeros([nums, 448, 448, 3], dtype=np.uint8)\n",
        "    xml = []\n",
        "    class_name = []\n",
        "    for idx, filename in enumerate(filenames):\n",
        "        imgname, gt_bbox, name_bbox = read_xml(xmlpath + filename)\n",
        "        img = np.array(Image.open(imgpath + imgname))\n",
        "        scaled_img, scaled_bbox = ToScaleImg(img, 448, 448, gt_bbox)\n",
        "        imgs[idx, :, :, :] = scaled_img\n",
        "        xml.append(scaled_bbox)\n",
        "        class_name.append(name_bbox)\n",
        "        print(idx)\n",
        "    sio.savemat(\"pascal.mat\", {\"imgs\": imgs, \"bboxes\": xml, \"class\": class_name})\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PWX_84mop5af"
      },
      "source": [
        "#training model\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "BATCH_SIZE = 64\n",
        "img_path = \"/content/VOC/VOCdevkit/VOC2007/JPEGImages\"\n",
        "xml_path = \"/content/VOC/VOCdevkit/VOC2007/Annotations\"\n",
        "\n",
        "\n",
        "def train():\n",
        "    inputs = tf.placeholder(tf.float32, [None, 448, 448, 3])\n",
        "    labels = tf.placeholder(tf.float32, [None, 7, 7, 25])\n",
        "    prediction = vgg16(inputs)\n",
        "    loss, loss_bboxes, loss_confidence_obj, loss_confidence_noobj, loss_class = yolo_loss(prediction, labels)\n",
        "    Opt = tf.train.MomentumOptimizer(1e-3, momentum=0.9).minimize(loss)\n",
        "    sess = tf.Session()\n",
        "    sess.run(tf.global_variables_initializer())\n",
        "    saver = tf.train.Saver()\n",
        "    # saver.restore(sess, \"./save_para/.\\\\model.ckpt\")\n",
        "    for i in range(10000):\n",
        "        batch_img, batch_labels = read_batch(img_path, xml_path, BATCH_SIZE)\n",
        "        sess.run(Opt, feed_dict={inputs: batch_img, labels: batch_labels})\n",
        "        [LOSS, b, o, n, c] = sess.run([loss, loss_bboxes, loss_confidence_obj, loss_confidence_noobj, loss_class], feed_dict={inputs: batch_img, labels: batch_labels})\n",
        "        print(\"Iteration: %d, Loss: %f, loss_bbox: %f, loss_confi_obj: %f, loss_confi_noobj: %f, loss_class: %f\"%(i, LOSS, b, o, n, c))\n",
        "        if i % 500 == 0:\n",
        "            saver.save(sess, \"./save_para/model.ckpt\")\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    train()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bhhoOjTEp-Hu"
      },
      "source": [
        "#testing YOLO\n",
        "\n",
        "\n",
        "#\n",
        "# import os\n",
        "# os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
        "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\"\n",
        "\n",
        "BATCH_SIZE = 1\n",
        "img_path = \"/content/VOC/VOCdevkit/VOC2007/JPEGImages\"\n",
        "xml_path = \"/content/VOC/VOCdevkit/VOC2007/Annotations\"\n",
        "\n",
        "OBJECT_NAMES = [\"tvmonitor\", \"train\", \"sofa\", \"sheep\", \"cat\", \"chair\", \"bottle\", \"motorbike\", \"boat\", \"bird\",\n",
        "                   \"person\", \"aeroplane\", \"dog\", \"pottedplant\", \"cow\", \"bus\", \"diningtable\", \"horse\", \"bicycle\", \"car\"]\n",
        "\n",
        "def norm2bbox(norm_bbox, H=448, W=448, S=7):\n",
        "    temp = tf.constant([[0., 1., 2., 3., 4., 5., 6.]])\n",
        "    col = tf.tile(temp, multiples=[7, 1])[tf.newaxis, :, :, tf.newaxis, tf.newaxis]\n",
        "    row = tf.transpose(col, perm=[0, 2, 1, 3, 4])\n",
        "    x, y = norm_bbox[:, :, :, :, 0:1], norm_bbox[:, :, :, :, 1:2]\n",
        "    h, w = norm_bbox[:, :, :, :, 2:3], norm_bbox[:, :, :, :, 3:4]\n",
        "    x = (x + col) * (W / S)\n",
        "    y = (y + row) * (H / S)\n",
        "    h = tf.square(h) * H\n",
        "    w = tf.square(w) * W\n",
        "    x1 = x - w/2\n",
        "    y1 = y - h/2\n",
        "    x2 = x + w/2\n",
        "    y2 = y + h/2\n",
        "    coord = tf.concat([x1, y1, x2, y2], axis=-1)\n",
        "    return coord\n",
        "\n",
        "def select_bbox_again(indx_class, scores_class, pred_bboxes):\n",
        "    #indx_class: the index after non-maximun suppression, [20, 5]\n",
        "    #scores_class: confidence * class_confidence, [20, 98]\n",
        "    #pred_bboxes: [98, 4]\n",
        "    mask = np.zeros_like(scores_class)\n",
        "    for i in range(20):\n",
        "        for j in range(5):\n",
        "            mask[i, indx_class[i, j]] = 1\n",
        "    scores_class[scores_class < 0.2] = 0\n",
        "    scores_class *= mask\n",
        "    max_score = np.max(scores_class, axis=0)\n",
        "    indx_bboxes = np.arange(0, 98)\n",
        "    indx_bboxes = indx_bboxes[max_score > 0]\n",
        "    class_indx = np.argmax(scores_class, axis=0)\n",
        "    bbox_indx = class_indx[max_score > 0]\n",
        "    return pred_bboxes[indx_bboxes], bbox_indx\n",
        "\n",
        "def test(img):\n",
        "    inputs = tf.placeholder(tf.float32, [1, 448, 448, 3])\n",
        "    prediction = vgg16(inputs)\n",
        "    pred_bboxes = prediction[:, :, :, :8]#[1, 7, 7, 8]\n",
        "    pred_bboxes = tf.reshape(pred_bboxes, [1, 7, 7, 2, 4])#[1, 7, 7, 2, 4]\n",
        "    pred_confidence = prediction[:, :, :, 8:10]#[1, 7, 7, 2]\n",
        "    pred_class = prediction[:, :, :, 10:]#[1, 7, 7, 20]\n",
        "    pred_bboxes = norm2bbox(pred_bboxes)#[1, 7, 7, 2, 4]\n",
        "    class_confidences = tf.split(pred_class, num_or_size_splits=20, axis=-1)\n",
        "    pred_bboxes = tf.reshape(pred_bboxes, [-1, 4])#reshaping to feed into non-max suppression unit\n",
        "    indx_class = []\n",
        "    scores_class = []\n",
        "    for class_confidence in class_confidences:\n",
        "        scores = pred_confidence * class_confidence#[1, 7, 7, 2]\n",
        "        scores = tf.reshape(scores, [-1])\n",
        "        indx = tf.image.non_max_suppression(pred_bboxes, scores, 5)\n",
        "        indx_class.append(indx[tf.newaxis, :])\n",
        "        scores_class.append(scores[tf.newaxis, :])\n",
        "    indx_class = tf.concat(indx_class, axis=0)\n",
        "    scores_class = tf.concat(scores_class, axis=0)\n",
        "    sess = tf.Session()\n",
        "    sess.run(tf.global_variables_initializer())\n",
        "    #saver = tf.train.Saver()\n",
        "    #saver.restore(sess, \"./save_para/.\\\\model.ckpt\")\n",
        "\n",
        "    INDX_CLASS, PRED_BBOXES, SCORES_CLASS = sess.run([indx_class, pred_bboxes, scores_class], feed_dict={inputs: img[np.newaxis, :, :, :]})\n",
        "    bboxes, class_num = select_bbox_again(INDX_CLASS, SCORES_CLASS, PRED_BBOXES)\n",
        "    for i in range(bboxes.shape[0]):\n",
        "        try:\n",
        "            img = draw_bbox(img, np.int32(bboxes[i]), OBJECT_NAMES[class_num[i]])\n",
        "        except:\n",
        "            continue\n",
        "    Image.fromarray(np.uint8(img)).show()\n",
        "\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    img = np.array(Image.open(\"/content/drive/My Drive/keanu.jpg\").resize([448, 448]))\n",
        "    test(img)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JLRygyTZViG8"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}