notebooks/EXP1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Anchor Generation Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _extract_cor(anchor):\n",
    "    \"\"\"\n",
    "    Return width, height, x center, and y center for an anchor (window).\n",
    "    \"\"\"\n",
    "    # anchor is 1X4\n",
    "    w = anchor[2] - anchor[0] + 1\n",
    "    h = anchor[3] - anchor[1] + 1\n",
    "    x_ctr = anchor[0] + 0.5 * (w - 1)\n",
    "    y_ctr = anchor[1] + 0.5*(w-1)\n",
    "    return w,h,x_ctr,y_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _make_anchors(ws,hs,x_ctr,y_ctr):\n",
    "    \"\"\"\n",
    "    Given a vector of widths (ws) and heights (hs) around a center (x_ctr,y_ctr),\n",
    "    Return a set of anchor boxes in (w, h, x_ctr, y_ctr) format.\n",
    "    \"\"\"\n",
    "    # ws,hs,x_ctr,y_ctr are numpy arrays\n",
    "    w = ws[:,np.newaxis] # [1,4] -> [4,1] i.e it generate array of one more dimension\n",
    "    # print('w in mkanchors: ',w)\n",
    "    h = hs[:,np.newaxis] # [1,4] -> [4,1]\n",
    "    # print('h in mkanchors: ',h)\n",
    "    anchors = np.hstack((x_ctr - 0.5*(w-1),\n",
    "                                            y_ctr - 0.5*(h-1),\n",
    "                                            x_ctr + 0.5*(w-1),\n",
    "                                            y_ctr + 0.5*(h-1))) # horizontal stack\n",
    "    # print('anchors in mkanchors: ',anchors)\n",
    "    return anchors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _ratio_enum(anchor,ratios):\n",
    "    \"\"\"\n",
    "    Enumerate a set of anchors for each aspect ratio wrt an anchor.\n",
    "    \"\"\"\n",
    "    \n",
    "    w,h,x_ctr,y_ctr = _extract_cor(anchor)\n",
    "    # print(w,h,x_ctr,y_ctr)\n",
    "    size = w*h\n",
    "    # print('size='+str(size))\n",
    "    size_ratios = size/ratios\n",
    "    # print('size_ratios='+str(size_ratios))\n",
    "    ws = np.round(np.sqrt(size_ratios))\n",
    "    # print('ws='+str(ws))\n",
    "    hs = np.round(ws*ratios)\n",
    "    # print('hs='+str(hs))\n",
    "    anchors = _make_anchors(ws,hs,x_ctr,y_ctr)\n",
    "    # print('anchors in ratio_enu='+str(anchors))\n",
    "    return anchors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _scale_enum(anchor,scales):\n",
    "    \"\"\"\n",
    "    Enumerate a set of anchors for each scale wrt an anchor.\n",
    "    \"\"\"\n",
    "    w, h, x_ctr, y_ctr = _extract_cor(anchor)\n",
    "    # print(\"anchor in scle_enum=\"+str(anchor))\n",
    "    ws = w*scales\n",
    "    hs = h*scales\n",
    "    anchors = _make_anchors(ws,hs,x_ctr,y_ctr)\n",
    "    # print(\"Final anchors in scale_enum=\"+str(anchors))\n",
    "    return anchors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  It generates 9 anchor boxes from a base anchor box\n",
    "\n",
    "def generate_anchors(base_size=16,ratios=[0.5,1,2],scales=np.array([8,16,32])):\n",
    "    base_anchor = np.array([1,1,base_size,base_size]) - 1\n",
    "    # print(base_anchor)\n",
    "    ratio_anchors = _ratio_enum(base_anchor,ratios)\n",
    "    # print(ratio_anchors.shape)\n",
    "    anchors_list=[]\n",
    "    for i in range(ratio_anchors.shape[0]):\n",
    "        anc = _scale_enum(ratio_anchors[i,:],scales)\n",
    "        anchors_list.append(anc)\n",
    "    anchors = np.vstack(anchors_list)\n",
    "    return anchors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -84. ,  -34.5,   99. ,   60.5],\n",
       "       [-176. ,  -82.5,  191. ,  108.5],\n",
       "       [-360. , -178.5,  375. ,  204.5],\n",
       "       [ -56. ,  -56. ,   71. ,   71. ],\n",
       "       [-120. , -120. ,  135. ,  135. ],\n",
       "       [-248. , -248. ,  263. ,  263. ],\n",
       "       [ -36. ,  -85.5,   51. ,   89.5],\n",
       "       [ -80. , -173.5,   95. ,  177.5],\n",
       "       [-168. , -349.5,  183. ,  353.5]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_anchors()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create uniformly spaced grid with spacing equal to stride\n",
    "\n",
    "def generate_anchors_pre_tf(height, width, feat_stride=16, anchor_scales=(8,16,32), anchor_ratios=(0.5,1,2)):\n",
    "    \"\"\"\n",
    "    A wrapper function to generate anchors given different scales and\n",
    "    ratios.\n",
    "    \"\"\"\n",
    "\n",
    "    shift_x = tf.range(width) * feat_stride # [0,16,32,48] width\n",
    "    shift_y = tf.range(height) * feat_stride # [0,16,32,48] height\n",
    "    shift_x, shift_y = tf.meshgrid(shift_x, shift_y) # meshgrid cols, rows , meshgrid generates a grid of points in ND space\n",
    "    # meshgrid enumerate shift_x row wise and shift_y col wise\n",
    "    shift_x = tf.reshape(shift_x, shape=(-1,)) # reshape to 1D\n",
    "    shift_y = tf.reshape(shift_y, shape=(-1,))\n",
    "    shifts = tf.stack((shift_x, shift_y, shift_x, shift_y), axis=1) # vertical stack by row\n",
    "    K = tf.multiply(width, height)\n",
    "    shifts = tf.transpose(tf.reshape(shifts,shape=[1,K,4]),perm=[1,0,2]) # reshaping into Kx1x4\n",
    "\n",
    "    anchors = generate_anchors(ratios=np.array(anchor_ratios), scales=np.array(anchor_scales)) # basic 9 anchor boxes of shape (9,4)\n",
    "    A = anchors.shape[0] # 9\n",
    "    anchor_constants = tf.constant(anchors.reshape((1, A, 4)), dtype=tf.int32) # reshape to 1x9x4\n",
    "    \n",
    "    length = K*A\n",
    "    anchors_tf = tf.reshape(tf.add(anchor_constants,shifts),shape=[length,4]) # add shift to anchors element wise\n",
    "    return tf.cast(anchors_tf,tf.float32),length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type of tensor_anchors:  <class 'tensorflow.python.framework.ops.EagerTensor'>\n",
      "tensor_anchors shape (16650, 4)\n",
      "length=tf.Tensor(16650, shape=(), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "tensor_anchors, length = generate_anchors_pre_tf(height=600//16,width=800//16)\n",
    "print(\"type of tensor_anchors: \",type(tensor_anchors))\n",
    "print(\"tensor_anchors shape\",tensor_anchors.shape)\n",
    "print(\"length=\"+str(length))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For Bounding box regression coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculating bounding box regression coefficients\n",
    "def bbox_transform(original_rois,gt_rois):\n",
    "    original_widths = original_rois[:,2] - original_rois[:,0] + 1.0\n",
    "    original_heights = original_rois[:,3] - original_rois[:,1] + 1.0\n",
    "    original_ctr_x = original_rois[:,0] + 0.5 * original_widths\n",
    "    original_ctr_y = original_rois[:,1] + 0.5 * original_heights\n",
    "\n",
    "    gt_widths = gt_rois[:,2] - gt_rois[:,0] + 1.0\n",
    "    gt_heights = gt_rois[:,3] - gt_rois[:,1] + 1.0\n",
    "    gt_ctr_x = gt_rois[:,0] + 0.5 * gt_widths\n",
    "    gt_ctr_y = gt_rois[:,1] + 0.5 * gt_heights\n",
    "\n",
    "    targets_dx = (gt_ctr_x - original_ctr_x) / original_widths\n",
    "    targets_dy = (gt_ctr_y - original_ctr_y) / original_heights\n",
    "    targets_dw = np.log(gt_widths / original_widths)\n",
    "    targets_dh = np.log(gt_heights / original_heights)\n",
    "\n",
    "    targets = np.vstack((targets_dx, targets_dy, targets_dw, targets_dh)).transpose()\n",
    "    return targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.05882353, 0.05882353, 0.        , 0.        ],\n",
       "       [0.03030303, 0.03030303, 0.        , 0.        ]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bbox_transform(np.array([[-1,-1,15,15],[-1,-1,31,31]]),np.array([[0,0,16,16],[0,0,32,32]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bbox_transform_inv_tf(boxes, deltas):\n",
    "    if boxes.shape[0] == 0:\n",
    "        return np.zeros((0, deltas.shape[1]), dtype=deltas.dtype)\n",
    "    \n",
    "    boxes = tf.cast(boxes, deltas.dtype)\n",
    "    Original_widths = boxes[:, 2] - boxes[:, 0] + 1.0\n",
    "    Original_heights = boxes[:, 3] - boxes[:, 1] + 1.0\n",
    "    Original_ctr_x = boxes[:, 0] + 0.5 * Original_widths\n",
    "    Original_ctr_y = boxes[:, 1] + 0.5 * Original_heights\n",
    "\n",
    "    targets_dx = deltas[:, 0::4]\n",
    "    targets_dy = deltas[:, 1::4]\n",
    "    targets_dw = deltas[:, 2::4]\n",
    "    targets_dh = deltas[:, 3::4]\n",
    "\n",
    "    pred_ctr_x = tf.add(tf.multiply(targets_dx, Original_widths), Original_ctr_x)\n",
    "    pred_ctr_y = tf.add(tf.multiply(targets_dy, Original_heights), Original_ctr_y)\n",
    "    pred_w = tf.multiply(tf.exp(targets_dw), Original_widths)\n",
    "    pred_h = tf.multiply(tf.exp(targets_dh), Original_heights)\n",
    "\n",
    "    pred_boxes0 = tf.subtract(pred_ctr_x,pred_w*0.5)\n",
    "    pred_boxes1 = tf.subtract(pred_ctr_y,pred_h*0.5)\n",
    "    pred_boxes2 = tf.add(pred_ctr_x,pred_w*0.5)\n",
    "    pred_boxes3 = tf.add(pred_ctr_y,pred_h*0.5)\n",
    "\n",
    "    predicted_boxes = tf.stack([pred_boxes0,pred_boxes1,pred_boxes2,pred_boxes3],axis=1)\n",
    "    return predicted_boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clip_boxes_tf(boxes, im_info):\n",
    "    \"\"\"\n",
    "    Clip boxes to image boundaries.\n",
    "    boxes: [N, 4* num_classes]\n",
    "    im_info: [image_height, image_width, scale_ratios]\n",
    "    \"\"\"\n",
    "    # x1 >= 0\n",
    "    boxes[:, 0::4] = tf.maximum(tf.minimum(boxes[:, 0::4], im_info[1] - 1), 0)\n",
    "    # y1 >= 0\n",
    "    boxes[:, 1::4] = tf.maximum(tf.minimum(boxes[:, 1::4], im_info[0] - 1), 0)\n",
    "    # x2 < im_info[1]\n",
    "    boxes[:, 2::4] = tf.maximum(tf.minimum(boxes[:, 2::4], im_info[1] - 1), 0)\n",
    "    # y2 < im_info[0]\n",
    "    boxes[:, 3::4] = tf.maximum(tf.minimum(boxes[:, 3::4], im_info[0] - 1), 0)\n",
    "    return boxes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NMS - Non-Maximum Suppression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Head Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.applications.vgg16 import VGG16\n",
    "from tensorflow.keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_network = VGG16(weights='imagenet', include_top=False, input_shape=(600,800,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# top 18 layers of vgg16 for head network\n",
    "head_network = Model(inputs=head_network.input, outputs=head_network.get_layer('block5_conv3').output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 600, 800, 3)]     0         \n",
      "                                                                 \n",
      " block1_conv1 (Conv2D)       (None, 600, 800, 64)      1792      \n",
      "                                                                 \n",
      " block1_conv2 (Conv2D)       (None, 600, 800, 64)      36928     \n",
      "                                                                 \n",
      " block1_pool (MaxPooling2D)  (None, 300, 400, 64)      0         \n",
      "                                                                 \n",
      " block2_conv1 (Conv2D)       (None, 300, 400, 128)     73856     \n",
      "                                                                 \n",
      " block2_conv2 (Conv2D)       (None, 300, 400, 128)     147584    \n",
      "                                                                 \n",
      " block2_pool (MaxPooling2D)  (None, 150, 200, 128)     0         \n",
      "                                                                 \n",
      " block3_conv1 (Conv2D)       (None, 150, 200, 256)     295168    \n",
      "                                                                 \n",
      " block3_conv2 (Conv2D)       (None, 150, 200, 256)     590080    \n",
      "                                                                 \n",
      " block3_conv3 (Conv2D)       (None, 150, 200, 256)     590080    \n",
      "                                                                 \n",
      " block3_pool (MaxPooling2D)  (None, 75, 100, 256)      0         \n",
      "                                                                 \n",
      " block4_conv1 (Conv2D)       (None, 75, 100, 512)      1180160   \n",
      "                                                                 \n",
      " block4_conv2 (Conv2D)       (None, 75, 100, 512)      2359808   \n",
      "                                                                 \n",
      " block4_conv3 (Conv2D)       (None, 75, 100, 512)      2359808   \n",
      "                                                                 \n",
      " block4_pool (MaxPooling2D)  (None, 37, 50, 512)       0         \n",
      "                                                                 \n",
      " block5_conv1 (Conv2D)       (None, 37, 50, 512)       2359808   \n",
      "                                                                 \n",
      " block5_conv2 (Conv2D)       (None, 37, 50, 512)       2359808   \n",
      "                                                                 \n",
      " block5_conv3 (Conv2D)       (None, 37, 50, 512)       2359808   \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 14,714,688\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "head_network.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Proposal Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def proposal_layer(rpn_cls_prob, rpn_bbox_pred, im_info, _feat_stride, anchors, num_anchors):\n",
    "    pre_nms_topN = 12000\n",
    "    post_nms_topN = 2000\n",
    "    nms_thresh = 0.7\n",
    "\n",
    "    scores = rpn_cls_prob[:, :, :, num_anchors:]\n",
    "    scores = tf.reshape(scores, shape=(-1,))\n",
    "    rpn_bbox_pred = tf.reshape(rpn_bbox_pred, shape=(-1, 4))\n",
    "\n",
    "    proposals = bbox_transform_inv_tf(anchors, rpn_bbox_pred)\n",
    "    proposals = clip_boxes_tf(proposals, im_info[:2])\n",
    "\n",
    "    indices = tf.image.non_max_suppression(proposals, scores, max_output_size=post_nms_topN, iou_threshold=nms_thresh)\n",
    "    boxes = tf.gather(proposals, indices)\n",
    "    boxes = tf.to_float(boxes)\n",
    "    scores = tf.gather(scores, indices)\n",
    "    scores = tf.reshape(scores, shape=(-1, 1))\n",
    "\n",
    "    batch_inds = tf.zeros((tf.shape(indices)[0],1), dtype=tf.float32)\n",
    "    rois = tf.concat([batch_inds, boxes], axis=1)\n",
    "\n",
    "    return rois, scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RPN network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tf_slim as slim\n",
    "\n",
    "initializer = tf.random_normal_initializer(mean=0.0, stddev=0.01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _region_proposal(net_conv,is_training,initializer):\n",
    "    rpn = slim.conv2d(net_conv,512,[3,3],trainable=is_training,weights_initializer=initializer,scope='rpn_conv/3x3')\n",
    "    rpn_cls_score = slim.conv2d(rpn,num_anchors*2,[1,1],trainable=is_training,weights_initializer=initializer,\n",
    "                                padding='VALID',activation_fn=None,scope='rpn_cls_score')\n",
    "    rpn_bbox_pred = slim.conv2d(rpn,*4,[1,1],trainable=is_training,weights_initializer=initializer,\n",
    "                                padding='VALID',activation_fn=None,scope='rpn_bbox_pred')\n",
    "    rpn_cls_prob = tf.nn.softmax(rpn_cls_score)\n",
    "    rpn_cls_prob = tf.reshape(rpn_cls_prob,[-1,2])\n",
    "    rpn_bbox_pred = tf.reshape(rpn_bbox_pred,[-1,4])\n",
    "    return rpn_cls_prob,rpn_bbox_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Anchor Target Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import numpy.random as npr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bbox_overlaps(boxes,query_boxes):\n",
    "    \"\"\"\n",
    "    Parameters:\n",
    "    ----------\n",
    "    boxes: numpy array (N,4)\n",
    "    query_boxes: numpy array (K,4)\n",
    "    Returns:\n",
    "    -------\n",
    "    overlaps: numpy array (N,K)\n",
    "    \"\"\"\n",
    "    N = boxes.shape[0]\n",
    "    K = query_boxes.shape[0]\n",
    "    overlaps = np.zeros((N,K),dtype=np.float)\n",
    "    for k in range(K):\n",
    "        box_area = (\n",
    "            (query_boxes[k, 2] - query_boxes[k, 0] + 1) *\n",
    "            (query_boxes[k, 3] - query_boxes[k, 1] + 1)\n",
    "        )\n",
    "        for n in range(N):\n",
    "            iw = (\n",
    "                min(boxes[n, 2], query_boxes[k, 2]) -\n",
    "                max(boxes[n, 0], query_boxes[k, 0]) + 1\n",
    "            )\n",
    "            if iw > 0:\n",
    "                ih = (\n",
    "                    min(boxes[n, 3], query_boxes[k, 3]) -\n",
    "                    max(boxes[n, 1], query_boxes[k, 1]) + 1\n",
    "                )\n",
    "                if ih > 0:\n",
    "                    ua = float((boxes[n, 2] - boxes[n, 0] + 1) *\n",
    "                           (boxes[n, 3] - boxes[n, 1] + 1) + box_area - iw * ih)\n",
    "                    overlaps[n, k] = iw * ih / ua\n",
    "    return overlaps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1.         0.14285714 0.         1.        ]\n",
      " [0.14285714 1.         0.         0.14285714]\n",
      " [0.         0.         1.         0.        ]]\n",
      "[0 1 2 0]\n"
     ]
    }
   ],
   "source": [
    "## call bbox_overlaps\n",
    "boxes = np.array([[0,0,1,1],[1,1,2,2],[3,3,4,4]])\n",
    "query_boxes = np.array([[0,0,1,1],[1,1,2,2],[3,3,4,4],[0,0,1,1]])\n",
    "overlaps = bbox_overlaps(boxes,query_boxes)\n",
    "print(overlaps)\n",
    "argmax_overlaps = overlaps.argmax(axis=0)\n",
    "print(argmax_overlaps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "RPN_NEGATIVE_OVERLAP = 0.3\n",
    "RPN_POSITIVE_OVERLAP = 0.7\n",
    "RPN_BATCHSIZE = 256\n",
    "RPN_FG_FRACTION = 0.5\n",
    "RPN_BBOX_INSIDE_WEIGHTS = (1.0, 1.0, 1.0, 1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def anchor_target_layer(rpn_cls_score,gt_boxes,im_info,_feat_stride,all_anchors,num_anchors):\n",
    "    A = num_anchors\n",
    "    total_anchors = all_anchors.shape[0]\n",
    "    K = total_anchors / num_anchors\n",
    "\n",
    "    # allow boxes to sit over the edge by a small amount\n",
    "    _allowed_border = 0\n",
    "\n",
    "    # map of shape (..., H, W)\n",
    "    height, width = rpn_cls_score.shape[1:3]\n",
    "\n",
    "    # only keep anchors inside the image\n",
    "    inds_inside = np.where(\n",
    "        (all_anchors[:, 0] >= -_allowed_border) &\n",
    "        (all_anchors[:, 1] >= -_allowed_border) &\n",
    "        (all_anchors[:, 2] < im_info[1] + _allowed_border) &  # width\n",
    "        (all_anchors[:, 3] < im_info[0] + _allowed_border)  # height\n",
    "    )[0]\n",
    "\n",
    "    # keep only inside anchors\n",
    "    anchors = all_anchors[inds_inside, :]\n",
    "\n",
    "    # label: 1 is positive, 0 is negative, -1 is dont care\n",
    "    labels = np.empty((len(inds_inside),), dtype=np.float32)\n",
    "    labels.fill(-1)\n",
    "\n",
    "    # overlaps between the anchors and the gt boxes\n",
    "    # overlaps (ex, gt)\n",
    "    overlaps = bbox_overlaps(\n",
    "        np.ascontiguousarray(anchors, dtype=np.float),\n",
    "        np.ascontiguousarray(gt_boxes, dtype=np.float))\n",
    "    argmax_overlaps = overlaps.argmax(axis=1)\n",
    "    max_overlaps = overlaps[np.arange(len(inds_inside)), argmax_overlaps]\n",
    "    gt_argmax_overlaps = overlaps.argmax(axis=0)\n",
    "    gt_max_overlaps = overlaps[gt_argmax_overlaps,\n",
    "                                 np.arange(overlaps.shape[1])]\n",
    "    gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0]\n",
    "\n",
    "    labels[max_overlaps < RPN_NEGATIVE_OVERLAP] = 0\n",
    "    labels[gt_argmax_overlaps] = 1\n",
    "    labels[max_overlaps >= RPN_POSITIVE_OVERLAP] = 1\n",
    "\n",
    "    # subsample positive labels if we have too many\n",
    "    num_fg = int(RPN_FG_FRACTION * RPN_BATCHSIZE)\n",
    "    fg_inds = np.where(labels == 1)[0]\n",
    "    if len(fg_inds) > num_fg:\n",
    "        disable_inds = npr.choice(\n",
    "            fg_inds, size=(len(fg_inds) - num_fg), replace=False)\n",
    "        labels[disable_inds] = -1\n",
    "    \n",
    "    # subsample negative labels if we have too many\n",
    "    num_bg = RPN_BATCHSIZE - np.sum(labels == 1)\n",
    "    bg_inds = np.where(labels == 0)[0]\n",
    "    if len(bg_inds) > num_bg:\n",
    "        disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False)\n",
    "        labels[disable_inds] = -1\n",
    "    \n",
    "    bbox_targets = np.zeros((len(inds_inside), 4), dtype=np.float32)\n",
    "    bbox_targets = _compute_targets(anchors, gt_boxes[argmax_overlaps, :])\n",
    "\n",
    "    bbox_inside_weights = np.zeros((len(inds_inside), 4), dtype=np.float32)\n",
    "    bbox_inside_weights[labels == 1, :] = np.array(RPN_BBOX_INSIDE_WEIGHTS)\n",
    "\n",
    "    bbox_outside_weights = np.zeros((len(inds_inside), 4), dtype=np.float32)\n",
    "    \n",
    "    bbox_outside_weights[labels == 1, :] = 1.0\n",
    "    bbox_outside_weights[labels == 0, :] = 1.0\n",
    "\n",
    "    # map up to original set of anchors\n",
    "    labels = _unmap(labels, total_anchors, inds_inside, fill=-1)\n",
    "    bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, fill=0)\n",
    "    bbox_inside_weights = _unmap(bbox_inside_weights, total_anchors, inds_inside, fill=0)\n",
    "    bbox_outside_weights = _unmap(bbox_outside_weights, total_anchors, inds_inside, fill=0)\n",
    "\n",
    "    # labels\n",
    "    labels = labels.reshape((1, height, width, A)).transpose(0, 3, 1, 2)\n",
    "    labels = labels.reshape((1, 1, A * height, width))\n",
    "    rpn_labels = labels\n",
    "\n",
    "    # bbox_targets\n",
    "    bbox_targets = bbox_targets.reshape((1, height, width, A * 4))\n",
    "    \n",
    "    rpn_bbox_targets = bbox_targets\n",
    "\n",
    "    # bbox_inside_weights\n",
    "    bbox_inside_weights = bbox_inside_weights.reshape((1, height, width, A * 4))\n",
    "    rpn_bbox_inside_weights = bbox_inside_weights\n",
    "\n",
    "    # bbox_outside_weights\n",
    "    bbox_outside_weights = bbox_outside_weights.reshape((1, height, width, A * 4))\n",
    "    rpn_bbox_outside_weights = bbox_outside_weights\n",
    "\n",
    "    return rpn_labels, rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _unmap(data, count, inds, fill=0):\n",
    "    \"\"\" Unmap a subset of item (data) back to the original set of items (of size count) \"\"\"\n",
    "    if len(data.shape) == 1:\n",
    "        ret = np.empty((count,), dtype=np.float32)\n",
    "        ret.fill(fill)\n",
    "        ret[inds] = data\n",
    "    else:\n",
    "        ret = np.empty((count,) + data.shape[1:], dtype=np.float32)\n",
    "        ret.fill(fill)\n",
    "        ret[inds, :] = data\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _compute_targets(ex_rois, gt_rois):\n",
    "    \"\"\"Compute bounding-box regression targets for an image.\"\"\"\n",
    "\n",
    "    assert ex_rois.shape[0] == gt_rois.shape[0]\n",
    "    assert ex_rois.shape[1] == 4\n",
    "    assert gt_rois.shape[1] == 5\n",
    "\n",
    "    return bbox_transform(ex_rois, gt_rois[:, :4]).astype(np.float32, copy=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.]], dtype=float32)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# call _compute_targets()\n",
    "anchors = np.array([[0, 0, 10, 10],[0, 0, 20, 20],[0, 0, 30, 30]])\n",
    "gt_boxes = np.array([[0, 0, 10, 10, 1],[0, 0, 20, 20, 1],[0, 0, 30, 30, 1]])\n",
    "_compute_targets(anchors, gt_boxes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Proposal Target Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "FG_FRACTION = 0.25\n",
    "BBOX_INSIDE_WEIGHTS = (1.0, 1.0, 1.0, 1.0)\n",
    "BG_THRESH_HI = 0.5\n",
    "BG_THRESH_LO = 0.1\n",
    "FG_THRESH = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def proposal_target_layer(rpn_rois,rpn_score,gt_boxes,_num_classes):\n",
    "    \"\"\"\n",
    "    Assign object detection proposals to ground-truth targets. Produces proposal\n",
    "    classification labels and bounding-box regression targets.\n",
    "    \"\"\"\n",
    "\n",
    "    # Proposal ROIs (0, x1, y1, x2, y2) coming from RPN\n",
    "    all_rois = rpn_rois\n",
    "    all_scores = rpn_score\n",
    "\n",
    "    # Include ground-truth boxes in the set of candidate rois\n",
    "    zeros = np.zeros((gt_boxes.shape[0], 1), dtype=gt_boxes.dtype)\n",
    "    all_rois = np.vstack(\n",
    "        (all_rois, np.hstack((zeros, gt_boxes[:, :-1])))\n",
    "    )\n",
    "    all_scores = np.vstack((all_scores, zeros))\n",
    "\n",
    "    num_images = 1\n",
    "    rois_per_image = BATCH_SIZE / num_images\n",
    "    fg_rois_per_image = np.round(FG_FRACTION * rois_per_image)\n",
    "\n",
    "    # Sample rois with classification labels and bounding box regression targets\n",
    "    labels, rois, roi_scores, bbox_targets, bbox_inside_weights = _sample_rois(\n",
    "        all_rois, all_scores, gt_boxes, fg_rois_per_image, rois_per_image, _num_classes)\n",
    "    \n",
    "    rois = rois.reshape(-1, 5)\n",
    "    roi_scores = roi_scores.reshape(-1)\n",
    "    labels = labels.reshape(-1, 1)\n",
    "    bbox_targets = bbox_targets.reshape(-1, _num_classes * 4)\n",
    "    bbox_inside_weights = bbox_inside_weights.reshape(-1, _num_classes * 4)\n",
    "    bbox_outside_weights = np.array(bbox_inside_weights > 0).astype(np.float32)\n",
    "\n",
    "    return rois, roi_scores, labels, bbox_targets, bbox_inside_weights, bbox_outside_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _get_bbox_regression_labels(bbox_target_data,num_classes):\n",
    "    \"\"\"Bounding-box regression targets (bbox_target_data) are stored in a\n",
    "    compact form N x (class, tx, ty, tw, th)\n",
    "\n",
    "    This function expands those targets into the 4-of-4*K representation used\n",
    "    by the network (i.e. only one class has non-zero targets).\n",
    "\n",
    "    Returns:\n",
    "        bbox_target (ndarray): N x 4K blob of regression targets\n",
    "        bbox_inside_weights (ndarray): N x 4K blob of loss weights\n",
    "    \"\"\"\n",
    "\n",
    "    clss = bbox_target_data[:, 0]\n",
    "    bbox_targets = np.zeros((clss.size, 4 * num_classes), dtype=np.float32)\n",
    "    bbox_inside_weights = np.zeros(bbox_targets.shape, dtype=np.float32)\n",
    "    inds = np.where(clss > 0)[0]\n",
    "    for ind in inds:\n",
    "        cls = clss[ind]\n",
    "        start = int(4 * cls)\n",
    "        end = start + 4\n",
    "        bbox_targets[ind, start:end] = bbox_target_data[ind, 1:]\n",
    "        bbox_inside_weights[ind, start:end] = BBOX_INSIDE_WEIGHTS\n",
    "    return bbox_targets, bbox_inside_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _compute_targets_PTL(ex_rois,gt_rois,labels):\n",
    "    \"\"\"Compute bounding-box regression targets for an image.\"\"\"\n",
    "\n",
    "    assert ex_rois.shape[0] == gt_rois.shape[0]\n",
    "    assert ex_rois.shape[1] == 4\n",
    "    assert gt_rois.shape[1] == 4\n",
    "\n",
    "    targets = bbox_transform(ex_rois, gt_rois).astype(np.float32, copy=False)\n",
    "    targets = np.hstack((labels[:, np.newaxis], targets)).astype(np.float32, copy=False)\n",
    "\n",
    "    return targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.,  0.,  0.,  0.,  0.,  0., 10., 10.,  0.,  0.,  0.,  0.,  0.,\n",
       "          0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0., 20., 20.,  0.,\n",
       "          0.,  0.,  0.],\n",
       "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0., 30., 30.,  0.,\n",
       "          0.,  0.,  0.]], dtype=float32),\n",
       " array([[0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0.]],\n",
       "       dtype=float32))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# call _get_bbox_regression_labels function\n",
    "bbox_target_data = np.array([[1,0,0,10,10],[2,0,0,20,20],[2,0,0,30,30]])\n",
    "num_classes = 4\n",
    "_get_bbox_regression_labels(bbox_target_data,num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _sample_rois(all_rois, all_scores, gt_boxes, fg_rois_per_image, rois_per_image, num_classes):\n",
    "    \"\"\" Generate a random sample of RoIs comprising foreground and background examples\n",
    "    \"\"\"\n",
    "    # overlaps: (rois x gt_boxes)\n",
    "    overlaps = bbox_overlaps(\n",
    "        np.ascontiguousarray(all_rois[:,1:5],dtype=np.float),\n",
    "        np.ascontiguousarray(gt_boxes[:,:4],dtype=np.float))\n",
    "    gt_assignment = overlaps.argmax(axis=1)\n",
    "    max_overlaps = overlaps.max(axis=1)\n",
    "    labels = gt_boxes[gt_assignment, 4]\n",
    "\n",
    "    # Select foreground RoIs as those with >= FG_THRESH overlap\n",
    "    fg_inds = np.where(max_overlaps >= FG_THRESH)[0]\n",
    "    # Select background RoIs as those within [BG_THRESH_LO,GB_THRESH_HI)\n",
    "    bg_inds = np.where((max_overlaps < BG_THRESH_HI) &\n",
    "                          (max_overlaps >= BG_THRESH_LO))[0]\n",
    "\n",
    "    # Ensure that a fixed number of regions are sampled\n",
    "    if fg_inds.size > 0 and bg_inds.size > 0:\n",
    "        fg_rois_per_image = min(fg_rois_per_image, fg_inds.size)\n",
    "        fg_inds = npr.choice(fg_inds, size=int(fg_rois_per_image), replace=False)\n",
    "        bg_rois_per_image = rois_per_image - fg_rois_per_image\n",
    "        to_replace = bg_inds.size < bg_rois_per_image\n",
    "        bg_inds = npr.choice(bg_inds, size=int(bg_rois_per_image), replace=to_replace)\n",
    "    elif fg_inds > 0:\n",
    "        to_replace = fg_inds.size < rois_per_image\n",
    "        fg_inds = npr.choice(fg_inds, size=int(rois_per_image), replace=to_replace)\n",
    "        fg_rois_per_image = rois_per_image\n",
    "    elif bg_inds > 0:\n",
    "        to_repace = bg_inds.size < rois_per_image\n",
    "        bg_inds = npr.choice(bg_inds, size=int(rois_per_image), replace=to_replace)\n",
    "        fg_rois_per_image = 0\n",
    "    \n",
    "    # the indices that we are selecting (both fg and bg)\n",
    "    keep_inds = np.append(fg_inds,bg_inds)\n",
    "    # Select sampled values from various arrays:\n",
    "    labels = labels[keep_inds]\n",
    "    # Clamp labels for the background RoIs to 0\n",
    "    labels[int(fg_rois_per_image):] = 0\n",
    "    rois = all_rois[keep_inds]\n",
    "    rois_scores = all_scores[keep_inds]\n",
    "\n",
    "    bbox_target_data = _compute_targets_PTL(\n",
    "        rois[:, 1:5], gt_boxes[gt_assignment[keep_inds], :4],labels)\n",
    "    \n",
    "    bbox_targets, bbox_inside_weights = _get_bbox_regression_labels(bbox_target_data,num_classes)\n",
    "\n",
    "    return labels, rois, rois_scores, bbox_targets, bbox_inside_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Crop Pooling Layer\n",
    "will implement this layer directly in Network."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from six.moves import range\n",
    "import PIL.Image as Image\n",
    "import PIL.ImageDraw as ImageDraw\n",
    "import PIL.ImageFont as ImageFont\n",
    "import PIL.ImageColor as ImageColor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "STANDARD_COLORS = [\n",
    "    'AliceBlue', 'Chartreuse', 'Aqua', 'Aquamarine', 'Azure', 'Beige', 'Bisque',\n",
    "    'BlanchedAlmond', 'BlueViolet', 'BurlyWood', 'CadetBlue', 'AntiqueWhite',\n",
    "    'Chocolate', 'Coral', 'CornflowerBlue', 'Cornsilk', 'Crimson', 'Cyan',\n",
    "    'DarkCyan', 'DarkGoldenRod', 'DarkGrey', 'DarkKhaki', 'DarkOrange',\n",
    "    'DarkOrchid', 'DarkSalmon', 'DarkSeaGreen', 'DarkTurquoise', 'DarkViolet',\n",
    "    'DeepPink', 'DeepSkyBlue', 'DodgerBlue', 'FireBrick', 'FloralWhite',\n",
    "    'ForestGreen', 'Fuchsia', 'Gainsboro', 'GhostWhite', 'Gold', 'GoldenRod',\n",
    "    'Salmon', 'Tan', 'HoneyDew', 'HotPink', 'IndianRed', 'Ivory', 'Khaki',\n",
    "    'Lavender', 'LavenderBlush', 'LawnGreen', 'LemonChiffon', 'LightBlue',\n",
    "    'LightCoral', 'LightCyan', 'LightGoldenRodYellow', 'LightGray', 'LightGrey',\n",
    "    'LightGreen', 'LightPink', 'LightSalmon', 'LightSeaGreen', 'LightSkyBlue',\n",
    "    'LightSlateGray', 'LightSlateGrey', 'LightSteelBlue', 'LightYellow', 'Lime',\n",
    "    'LimeGreen', 'Linen', 'Magenta', 'MediumAquaMarine', 'MediumOrchid',\n",
    "    'MediumPurple', 'MediumSeaGreen', 'MediumSlateBlue', 'MediumSpringGreen',\n",
    "    'MediumTurquoise', 'MediumVioletRed', 'MintCream', 'MistyRose', 'Moccasin',\n",
    "    'NavajoWhite', 'OldLace', 'Olive', 'OliveDrab', 'Orange', 'OrangeRed',\n",
    "    'Orchid', 'PaleGoldenRod', 'PaleGreen', 'PaleTurquoise', 'PaleVioletRed',\n",
    "    'PapayaWhip', 'PeachPuff', 'Peru', 'Pink', 'Plum', 'PowderBlue', 'Purple',\n",
    "    'Red', 'RosyBrown', 'RoyalBlue', 'SaddleBrown', 'Green', 'SandyBrown',\n",
    "    'SeaGreen', 'SeaShell', 'Sienna', 'Silver', 'SkyBlue', 'SlateBlue',\n",
    "    'SlateGray', 'SlateGrey', 'Snow', 'SpringGreen', 'SteelBlue', 'GreenYellow',\n",
    "    'Teal', 'Thistle', 'Tomato', 'Turquoise', 'Violet', 'Wheat', 'White',\n",
    "    'WhiteSmoke', 'Yellow', 'YellowGreen'\n",
    "]\n",
    "\n",
    "NUM_COLORS = len(STANDARD_COLORS)\n",
    "\n",
    "try:\n",
    "    FONT = ImageFont.truetype('arial.ttf', 24)\n",
    "except IOError:\n",
    "    FONT = ImageFont.load_default()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _draw_single_box(image, xmin, ymin, xmax, ymax, display_str, font, color='black', thickness=4):\n",
    "    draw = ImageDraw.Draw(image)\n",
    "    (left, right, top, bottom) = (xmin, xmax, ymin, ymax)\n",
    "    draw.line([(left, top), (left, bottom), (right, bottom),\n",
    "                (right, top), (left, top)], width=thickness, fill=color)\n",
    "    text_bottom = bottom\n",
    "    # Reverse list and print from bottom to top.\n",
    "    text_width, text_height = font.getsize(display_str)\n",
    "    margin = np.ceil(0.05 * text_height)\n",
    "    draw.rectangle([(left, text_bottom - text_height - 2 * margin), (left + text_width, text_bottom)], fill=color)\n",
    "    draw.text((left + margin, text_bottom - text_height - margin), display_str, fill='black', font=font)\n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_bounding_boxes(image,gt_boxes,im_info):\n",
    "    num_boxes = gt_boxes.shape[0]\n",
    "    gt_boxes_new = gt_boxes.copy()\n",
    "    gt_boxes_new[:,:4] = np.round(gt_boxes_new[:,:4]/copy() / im_info[2])\n",
    "    disp_image = Image.fromarray(np.uint8(image[0]))\n",
    "\n",
    "    for i in range(num_boxes):\n",
    "        this_class = int(gt_boxes_new[i,4])\n",
    "        disp_image = _draw_single_box(disp_image,gt_boxes_new[i,0],gt_boxes_new[i,1],gt_boxes_new[i,2],gt_boxes_new[i,3],'N%02d-C%02d'%(i,this_class),FONT,color=STANDARD_COLORS[this_class%NUM_COLORS])\n",
    "        image[0,:] = np.array(disp_image)\n",
    "        return image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network(object):\n",
    "    def __init__(self):\n",
    "        self._predictions = {}\n",
    "        self._losses = {}\n",
    "        self._anchor_targets = {}\n",
    "        self._proposal_targets={}\n",
    "        self._layers = {}\n",
    "        self._gt_image = None\n",
    "        self._act_summaries = []\n",
    "        self._score_summaries = {}\n",
    "        self._train_summaries = []\n",
    "        self._event_summaries = {}\n",
    "        self._variables_to_fix = {}\n",
    "\n",
    "    def _add_gt_image(self):\n",
    "        # add back mean\n",
    "        image - self._image + np.array([[[102.9801, 115.9465, 122.7717]]])\n",
    "        # BGR to RGB\n",
    "        resize = tf.image.resize(resized,self._im_info[:2],method=tf.image.ResizeMethod.BILINEAR)\n",
    "        self._gt_image = tf.reverse(resize,axis=[-1])\n",
    "    \n",
    "    def _add_gt_image_summary(self):\n",
    "        # use a customized visualization function to visualize the boxes\n",
    "        if self._gt_image is None:\n",
    "            self._add_gt_image()\n",
    "        image = tf.py_func(draw_bounding_boxes,\n",
    "                            [self._gt_image, self._gt_boxes, self._im_info],\n",
    "                            tf.float32, name=\"gt_boxes\")\n",
    "        return tf.summary.image('GROUND_TRUTH', image)\n",
    "\n",
    "    def _add_act_summary(self,tensor):\n",
    "        tf.summary.histogram('ACT/' + tensor.op.name + '/activations', tensor)\n",
    "        tf.summary.scalar('ACT/' + tensor.op.name + '/zero_fraction',tf.nn.zero_fraction(tensor))\n",
    "    \n",
    "    def _add_score_summary(self,key,tensor):\n",
    "        tf.summary.histogram('SCORE/' + tensor.op.name + '/' + key + '/scores', tensor)\n",
    "    \n",
    "    def _add_train_summary(self,var):\n",
    "        tf.summary.histogram('TRAIN/' + var.op.name, var)\n",
    "    \n",
    "    def _reshape_layer(self,bottom,num_dim, name):\n",
    "        input_shape = tf.shape(bottom)\n",
    "        with tf.variable_scope(name) as scope:\n",
    "            # change the channel to the caffe format\n",
    "            to_caffe = tf.transpose(bottom, [0, 3, 1, 2]) # NCHW where N is batch size, C is channel, H is height, W is width\n",
    "            # then force it to have channel 2\n",
    "            reshaped = tf.reshape(to_caffe, tf.concat(axis=0, values =[[1, num_dim, -1], [input_shape[2]]]))\n",
    "            # then swap the channel back\n",
    "            to_tf = tf.transpose(reshaped, [0, 2, 3, 1])\n",
    "            return to_tf\n",
    "\n",
    "    def _softmax_layer(self,bottom,name):\n",
    "        if name.startswith('rpn_cls_prob_reshape'):\n",
    "            input_shape = tf.shape(bottom)\n",
    "            bottom_reshaped = tf.reshape(bottom, [-1, input_shape[-1]])\n",
    "            reshaped_score = tf.nn.softmax(bottom_reshaped, name=name)\n",
    "            return tf.reshape(reshaped_score, input_shape)\n",
    "        return tf.nn.softmax(bottom, name=name)\n",
    "\n",
    "    def _proposal_layer(self, rpn_cls_prob, rpn_bbox_pred, name):\n",
    "        with tf.name_scope(name) as scope:\n",
    "            rois, rpn_scores = proposal_layer(rpn_cls_prob, rpn_bbox_pred, self.im_info, self._feat_stride, self._anchors, self._num_anchors)\n",
    "            rois.set_shape([None, 5])\n",
    "            rpn_scores.set_shape([None, 1])\n",
    "        return rois, rpn_scores\n",
    "\n",
    "    def _crop_pool_layer(self, bottom, rois, name):\n",
    "        with tf.name_scope(name) as scope:\n",
    "            batch_ids = tf.squeeze(tf.slice(rois, [0,0], [-1,1], name=\"batch_id\"), [1])\n",
    "            # Get the normalized coordinates of bounding boxes\n",
    "            bottom_shape = tf.shape(bottom)\n",
    "            height = (tf.cast(bottom_shape[1], tf.float32) - 1.) * np.float32(self._feat_stride[0])\n",
    "            width = (tf.cast(bottom_shape[2], tf.float32) - 1.) * np.float32(self._feat_stride[0])\n",
    "            x1 = tf.slice(rois, [0,1], [-1,1], name=\"x1\") / width\n",
    "            y1 = tf.slice(rois, [0,2], [-1,1], name=\"y1\") / height\n",
    "            x2 = tf.slice(rois, [0,3], [-1,1], name=\"x2\") / width\n",
    "            y2 = tf.slice(rois, [0,4], [-1,1], name=\"y2\") / height\n",
    "            bboxes = tf.stop_gradient(tf.concat([y1,x1,y2,x2], axis=1))\n",
    "            pre_pool_size = 7 * 2\n",
    "            crops = tf.image.crop_and_resize(bottom, bboxes, tf.cast(batch_ids,tf.int32), [pre_pool_size, pre_pool_size], name=\"crops\")\n",
    "        return slim.max_pool2d(crops, [2, 2], padding='SAME')\n",
    "\n",
    "        def _anchor_target_layer(self, rpn_cls_score, name):\n",
    "            with tf.name_scope(name) as scope:\n",
    "                rpn_labels, rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights = tf.py_func(_anchor_target_layer,\n",
    "                [rpn_cls_score, self._gt_boxes, self._im_info, self._feat_stride, self._anchors, self._num_anchors],\n",
    "                [tf.float32, tf.float32, tf.float32, tf.float32], name = \"anchor_target\")\n",
    "\n",
    "                rpn_labels.set_shape([1, 1, None, None])\n",
    "                rpn_bbox_targets.set_shape([1, None, None, self._num_anchors * 4])\n",
    "                rpn_bbox_inside_weights.set_shape([1, None, None, self._num_anchors*4])\n",
    "                rpn_bbox_outside_weights.set_shape([1, None, None, self._num_anchors*4])\n",
    "\n",
    "                rpn_labels = tf.cast(rpn_labels,tf.int32, name = 'to_int32')\n",
    "                self._anchor_targets['rpn_labels'] = rpn_labels\n",
    "                self._anchor_targets['rpn_bbox_targets'] = rpn_bbox_targets\n",
    "                self._anchor_targets['rpn_bbox_inside_weights'] = rpn_bbox_inside_weights\n",
    "                self._anchor_targets['rpn_bbox_outside_weights'] = rpn_bbox_outside_weights\n",
    "\n",
    "                self._score_summaries.update(self._anchor_targets)\n",
    "            return rpn_labels\n",
    "\n",
    "        def _proposal_target_layer(self, rois, roi_scores, name):\n",
    "            with tf.name_scope(name) as scope:\n",
    "                rois, roi_scores, labels, bbox_targets, bbox_inside_weights, bbox_outside_weights = tf.py_func(proposal_target_layer,\n",
    "                [rois, roi_scores, self._gt_boxes, self._num_classes],\n",
    "                [tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32],\n",
    "                name = \"proposal_target\")\n",
    "\n",
    "                rois.set_shape([128,5])\n",
    "                roi_scores.set_shape([128])\n",
    "                labels.set_shape([128,1])\n",
    "                bbox_targets.set_shape([128,self._num_classes*4])\n",
    "                bbox_inside_weights.set_shape([128,self._num_classes*4])\n",
    "                bbox_outside_weights.set_shape([128,self._num_classes*4])\n",
    "\n",
    "                self._proposal_targets['rois'] = rois\n",
    "                self._proposal_targets['labels'] = tf.cast(labels,tf.int32)\n",
    "                self._proposal_targets['bbox_targets'] = bbox_targets\n",
    "                self._proposal_targets['bbox_inside_weights'] = bbox_inside_weights\n",
    "                self._proposal_targets['bbox_outside_weights'] = bbox_outside_weights\n",
    "\n",
    "                self._score_summaries.update(self._proposal_targets)\n",
    "            return rois, roi_scores\n",
    "        \n",
    "        def _anchor_component(self):\n",
    "            with tf.name_scope('ANCHOR_'+self.tag) as scope:\n",
    "                # just to get the shape right\n",
    "                height = tf.cast((tf.math.ceil(self.im_info[0] / np.float32(self._feat_stride[0]))),tf.int32)\n",
    "                width = tf.cast((tf.math.ceil(self.im_info[1] / np.float32(self._feat_stride[0]))),tf.int32)\n",
    "                anchors, anchor_length = generate_anchors_pre_tf(height, width, self._feat_stride, self._anchor_scales, self._anchor_ratios)\n",
    "                anchors.set_shape([None, 4])\n",
    "                anchor_length.set_shape([])\n",
    "                self._anchors = anchors\n",
    "                self._anchor_length = anchor_length\n",
    "        \n",
    "        def _build_network(self,is_training=True):\n",
    "            initializer = tf.random.truncated_normal(0.0, 0.01)\n",
    "            initializer_bbox = tf.random.truncated_normal(0.0, 0.001)\n",
    "\n",
    "            new_conv = self._image_to_head(is_training)\n",
    "            with tf.name_scope(self._scope, self._scope):\n",
    "                self._anchor_component()\n",
    "                rois = self._region_proposal(net_conv,is_training,initializer)\n",
    "                pool5 = self._crop_pool_layer(net_conv, rois, \"pool5\")\n",
    "            \n",
    "            fc7 = self._head_to_tail(pool5, is_training)\n",
    "            with tf.name_scope(self._scope, self._scope):\n",
    "                cls_prob, bbox_pred = self._region_classification(fc7, is_training, initializer, initializer_bbox)\n",
    "            \n",
    "            self._score_summaries.update(self._predictions)\n",
    "\n",
    "            return rois, cls_prob, bbox_pred\n",
    "        \n",
    "        def _smooth_l1_loss(self, bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights, sigma=0.1, dim=[1]):\n",
    "            sigma_2 = sigma**2\n",
    "            box_diff = bbox_pred - bbox_targets\n",
    "            in_box_diff = bbox_inside_weights * box_diff\n",
    "            abs_in_box_diff = tf.math.abs(in_box_diff)\n",
    "            smoothL1_sign = tf.stop_gradient(tf.cast(tf.less(abs_in_box_diff,1.0 / sigma_2),tf.float32))\n",
    "            in_loss_box = tf.math.pow(in_box_diff,2) * (sigma_2 / 2.) * smoothL1_sign + (abs_in_box_diff - (0.5 / sigma_2)) * (1.0 - smoothL1_sign)\n",
    "            out_loss_box = bbox_outside_weights * in_loss_box\n",
    "            loss_box = tf.reduce_mean(tf.reduce_sum(out_loss_box, axis=dim))\n",
    "            return loss_box\n",
    "\n",
    "        def _region_proposal(self, net_conv, is_training, initializer):\n",
    "            rpn = slim.conv2d(net_conv, 512, [3,3], trainable=is_training, weights_initializer=initializer, scope=\"rpn_conv/3x3\")\n",
    "            self._act_summaries.append(rpn)\n",
    "            rpn_cls_score = slim.conv2d(rpn, self._num_anchors*2, [1,1], trainable=is_training, weights_initializer=initializer,padding='VALID', activation_fn=None, scope='rpn_cls_score')\n",
    "            rpn_cls_score_reshape = self._reshape_layer(rpn_cls_score, 2, 'rpn_cls_score_reshape')\n",
    "            rpn_cls_prob_reshape = self._softmax_layer(rpn_cls_score_reshape, \"rpn_cls_prob_reshape\")\n",
    "            rpn_cls_pred = tf.argmax(tf.reshape(rpn_cls_score_reshape, [-1, 2]), axis=1, name = \"rpn_cls_pred\")\n",
    "            rpn_cls_prob = self._reshape_layer(rpn_cls_prob_reshape, self._num_anchors*2, 'rpn_cls_prob')\n",
    "            rpn_bbox_pred = slim.conv2d(rpn, self._num_anchors*4, [1,1], trainable=is_training, weights_initializer=initializer, padding='VALID', activation_fn=None, scope='rpn_bbox_pred')\n",
    "            \n",
    "            if is_training:\n",
    "                rois, roi_scores = self._proposal_layer(rpn_cls_prob, rpn_bbox_pred, \"rois\")\n",
    "                rpn_labels = self._anchor_target_layer(rpn_cls_score, \"anchor\")\n",
    "                # Try to have a deterministic order for the computing graph, for reproducibility\n",
    "                with tf.control_dependencies([rpn_labels]):\n",
    "                    rois, _ = self._proposal_target_layer(rois, roi_scores, \"rpn_rois\")\n",
    "            \n",
    "            self._predictions[\"rpn_cls_score\"] = rpn_cls_score\n",
    "            self._predictions[\"rpn_cls_score_reshape\"] = rpn_cls_score_reshape\n",
    "            self._predictions[\"rpn_cls_prob\"] = rpn_cls_prob\n",
    "            self._predictions[\"rpn_cls_pred\"] = rpn_cls_pred\n",
    "            self._predictions[\"rpn_bbox_pred\"] = rpn_bbox_pred\n",
    "            self._predictions[\"rois\"] = rois\n",
    "\n",
    "            return rois\n",
    "\n",
    "        def _region_classification(self, fc7, is_training, initializer, initializer_bbox):\n",
    "            cls_score = slim.fully_connected(fc7, self._num_classes, weights_initializer= initializer, trainable= is_training, activation_fn=None, scope='cls_score')\n",
    "            cls_prob = self._softmax_layer(cls_score, \"cls_prob\")\n",
    "            cls_pred = tf.argmax(cls_score, axis=1, name=\"cls_pred\")\n",
    "            bbox_pred = slim.fully_connected(fc7, self._num_classes*4, weights_initializer= initializer_bbox, trainable= is_training, activation_fn=None, scope='bbox_pred')\n",
    "            self._predictions['cls_score'] = cls_score\n",
    "            self._predictions['cls_pred'] = cls_pred\n",
    "            self._predictions['cls_prob'] = cls_prob\n",
    "            self._predictions['bbox_pred'] = bbox_pred\n",
    "\n",
    "            return cls_prob, bbox_pred\n",
    "        \n",
    "        def _image_to_head(self,is_training,resuse=None):\n",
    "            raise NotImplementedError\n",
    "        def _head_to_tail(self,pool5,is_training,reuse=None):\n",
    "            raise NotImplementedError\n",
    "        \n",
    "\n",
    "        def create_architecture(self, mode, num_classes, tag=None, anchor_scales = (8,16,32), _anchor_ratios = (0.5,1,2)):\n",
    "            self._image = tf.compat.v1.placeholder(tf.float32, shape=[1, None, None, 3])\n",
    "            self._im_info = tf.compat.v1.placeholder(tf.float32, shape=[3])\n",
    "            self._gt_boxes = tf.compat.v1.placeholder(tf.float32, shape=[None, 5])\n",
    "            self._tag = tag\n",
    "\n",
    "            self._num_classes = num_classes\n",
    "            self._mode = mode\n",
    "            self._anchor_scales = anchor_scales\n",
    "            self._num_scales = len(anchor_scales)\n",
    "\n",
    "            self._anchor_ratios = _anchor_ratios\n",
    "            self._num_ratios = len(anchor_ratios)\n",
    "\n",
    "            self._num_anchors = self._num_scales * self._num_ratios\n",
    "\n",
    "            training = mode == 'TRAIN'\n",
    "\n",
    "            assert tag!=None\n",
    "\n",
    "            # handle most of the regularizers here\n",
    "            weights_regularizer = tf.keras.regularizers.l2(0.0001)\n",
    "            biases_regularizer = tf.compat.v1.no_regularizer\n",
    "\n",
    "            with arg_scope([slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected], weights_regularizer = weights_regularizer, biases_regularizer= biases_regularizer, biases_initializer = tf.constant_initializer(0.0)):\n",
    "                rois, cls_prob, bbox_pred = self._build_network(training)\n",
    "            layers_to_output = {'rois': rois}\n",
    "\n",
    "            for var in tf.Variable(trainable=True):\n",
    "                self._train_summaries.append(var)\n",
    "            \n",
    "            self._add_losses()\n",
    "            layers_to_output.update(self._losses)\n",
    "\n",
    "            val_summaries = []\n",
    "            with tf.device(\"/cpu:0\"):\n",
    "                val_summaries.append(self._add_gt_image_summary())\n",
    "                for key, var in self._event_summaries.items():\n",
    "                    val_summaries.append(tf.summary.scalar(key, var))\n",
    "                for key, var in self._score_summaries.items():\n",
    "                    self._add_score_summary(key, var)\n",
    "                for var in self._act_summaries:\n",
    "                    self._add_act_summary(var)\n",
    "                for var in self._train_summaries:\n",
    "                    self._add_train_summary(var)\n",
    "            \n",
    "            self._summary_op = tf.summary.merge_all()\n",
    "            self._summary_op_val = tf.summary.merge(val_summaries)\n",
    "\n",
    "            layers_to_output.update(self._predictions)\n",
    "            return layers_to_output\n",
    "\n",
    "        def train_step(self,sess,blobs,train_op):\n",
    "            feed_dict={self._image: blobs['data'], self._im_info: blobs['im_info'], self._gt_boxes: blobs['gt_boxes']}\n",
    "            rpn_loss_cls, rpn_loss_box, loss_cls, loss_box, loss, _ = sess.run([self._losses[\"rpn_cross_entropy\"], self._losses['rpn_loss_box'], self._losses['cross_entropy'], self._losses['loss_box'], self._losses['total_loss'], train_op], feed_dict=feed_dict)\n",
    "            return rpn_loss_cls, rpn_loss_box, loss_cls, loss_box, loss\n",
    "\n",
    "\n",
    "        def train_step_with_summary(self, sess, blobs, train_op):\n",
    "            feed_dict = {self._image: blobs['data'], self._im_info: blobs['im_info'], self._gt_boxes: blobs['gt_boxes']}\n",
    "            rpn_loss_cls, rpn_loss_box, loss_cls, loss_box, loss, summary, _ = sess.run([self._losses['rpn_cross_entropy'], self._losses['rpn_loss_box'], self._losses['cross_entropy'], self._losses['loss_box'], self._losses['total_loss'], self._summary_op, train_op], feed_dict=feed_dict)\n",
    "            return rpn_loss_cls, rpn_loss_box, loss_cls, loss_box, loss, summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VGG16 Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tf_slim as slim\n",
    "from tf_slim import losses\n",
    "from tf_slim import arg_scope\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "class vgg16(Network):\n",
    "    def __init__(self):\n",
    "        Network.__init__(self)\n",
    "        self._feat_stride = [16, ]\n",
    "        self._feat_compress = [1. / float(self._feat_stride[0]), ]\n",
    "        self._scope = 'vgg_16'\n",
    "    \n",
    "    def _image_to_head(self, is_training, reuse=None):\n",
    "        with tf.name_scope(self._scope, self._scope, reuse=reuse):\n",
    "            net = slim.repeat(self._image, 2, slim.conv2d, 64, [3, 3], trainable=False, scope='conv1')\n",
    "            net = slim.max_pool2d(net, [2, 2], padding='SAME', scope='pool1')\n",
    "            net = slim.repeat(net, 2, slim.conv2d, 128, [3,3], trainable=False, scope='conv2')\n",
    "            net = slim.max_pool2d(net, [2, 2], padding='SAME', scope='pool2')\n",
    "            net = slim.repeat(net, 3, slim.conv2d, 256, [3,3], trainable=is_training, scope='conv3')\n",
    "            net = slim.max_pool2d(net, [2, 2], padding='SAME', scope='pool3')\n",
    "            net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], trainable=is_training, scope='conv4')\n",
    "            net = slim.max_pool2d(net, [2, 2], padding='SAME', scope='pool4')\n",
    "            net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], trainable=is_training, scope='conv5')\n",
    "        \n",
    "        self._act_summaries.append(net)\n",
    "        self.layers['head'] = net\n",
    "\n",
    "        return net\n",
    "\n",
    "    def _head_to_tail(self,pool5,is_training, reuse=None):\n",
    "        with tf.name_scope(self._scope, self._scope, reuse=reuse):\n",
    "            pool5_flat = slim.flatten(pool5, scope='flatten')\n",
    "            fc6 = slim.fully_connected(pool5_flat, 4096, scope='fc6')\n",
    "            if is_training:\n",
    "                fc6 = slim.dropout(fc6, keep_prob=0.5, is_training=True, scope='dropout6')\n",
    "            fc7 = slim.fully_connected(fc6, 4096, scope='fc7')\n",
    "            if is_training:\n",
    "                fc7 = slim.dropout(fc7, keep_prob=0.5, is_training=True, scope='dropout7')\n",
    "        return fc7;\n",
    "\n",
    "    def get_variables_to_restore(self, variables, var_keep_dic):\n",
    "        variables_to_restore = []\n",
    "\n",
    "        for v in variables:\n",
    "            if v.name == (self._scope + '/fc6/weights:0') or v.name == (self._scope + '/fc7/weights:0'):\n",
    "                self._variables_to_fix[v.name] = v\n",
    "                continue\n",
    "\n",
    "            if v.name == (self._scope + 'conv1/conv1_1/weights:0'):\n",
    "                self._variables_to_fix[v.name] = v\n",
    "                continue\n",
    "\n",
    "            if v.name.split(':')[0] in var_keep_dic:\n",
    "                print('Variables restored: %s' % v.name)\n",
    "                variables_to_restore.append(v)\n",
    "        return variables_to_restore\n",
    "\n",
    "    def fix_variables(self,sess, pretrained_model):\n",
    "        print('Fix VGG16 layers..')\n",
    "        with tf.name_scope('Fix_VGG16') as scope:\n",
    "            with tf.device(\"/cpu:0\"):\n",
    "                fc6_conv = tf.compat.v1.get_variable(\"fc6_conv\", [7, 7, 512, 4096], trainable=False)\n",
    "                fc7_conv = tf.compat.v1.get_variable(\"fc7_conv\", [1, 1, 4096, 4096], trainable=False)\n",
    "                conv1_rgb = tf.compat.v1.get_variable(\"conv1_rgb\", [3, 3, 3, 64], trainable=False)\n",
    "                restorer_fc = tf.compat.v1.train.Saver({self._scope + \"/fc6/weights\": fc6_conv,\n",
    "                                                        self._scope + \"/fc7/weights\": fc7_conv,\n",
    "                                                        self._scope + \"/conv1/conv1_1/weights\": conv1_rgb})\n",
    "                restorer_fc.restore(sess, pretrained_model)\n",
    "                sess.run(tf.compat.v1.assign(self._variables_to_fix[self._scope + '/fc6/weights:0'], tf.reshape(fc6_conv,\n",
    "                                                                                                                 self._variables_to_fix[self._scope+'/fc6/weights:0'].get_shape())))\n",
    "                sess.run(tf.compat.v1.assign(self._variables_to_fix[self._scope + '/fc7/weights:0'], tf.reshape(fc7_conv,\n",
    "                                                                                                                    self._variables_to_fix[self._scope+'/fc7/weights:0'].get_shape())))\n",
    "                sess.run(tf.compat.v1.assign(self._variables_to_fix[self._scope + '/conv1/conv1_1/weights:0'], tf.reverse(conv1_rgb, [2])))                                                                                                    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### roidb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_roidb(imdb):\n",
    "    \"\"\"Enrich the imdb's roidb by adding some derived quantities that\n",
    "    are useful for training. This function precomputes the maximum\n",
    "    overlap, taken over ground-truth boxes, between each ROI and\n",
    "    each ground-truth box. The class with maximum overlap is also\n",
    "    recorded.\n",
    "    \"\"\"\n",
    "    roidb = imdb.roidb\n",
    "    if not (imdb.name.startswith('coco')):\n",
    "        sizes = [PIL.Image.open(imdb.image_path_at(i)).size\n",
    "                    for i in range(imdb.num_images)]\n",
    "    for i in range(len(imdb.image_index)):\n",
    "        roidb[i]['image'] = imdb.image_path_at(i)\n",
    "        if not (imdb.name.startswith('coco')):\n",
    "            roidb[i]['width'] = sizes[i][0]\n",
    "            roidb[i]['height'] = sizes[i][1]\n",
    "        # need gt_overlaps as a dense array for argmax\n",
    "        gt_overlaps = roidb[i]['gt_overlaps'].toarray()\n",
    "        # max overlap with gt over classes (columns)\n",
    "        max_overlaps = gt_overlaps.max(axis=1)\n",
    "        # gt class that had the max overlap\n",
    "        max_classes = gt_overlaps.argmax(axis=1)\n",
    "        roidb[i]['max_classes'] = max_classes\n",
    "        roidb[i]['max_overlaps'] = max_overlaps\n",
    "        # sanity checks\n",
    "        # max overlap of 0 => class should be zero (background)\n",
    "        zero_inds = np.where(max_overlaps == 0)[0]\n",
    "        assert all(max_classes[zero_inds] == 0)\n",
    "        # max overlap > 0 => class should not be zero (must be a fg class)\n",
    "        nonzero_inds = np.where(max_overlaps > 0)[0]\n",
    "        assert all(max_classes[nonzero_inds] != 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### blobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _im_list_to_blob(ims):\n",
    "    \"\"\" Convert a list of images into a network input.\n",
    "    Assumes images are already prepare (mean substracted, BGR order, ....).\n",
    "    \"\"\"\n",
    "\n",
    "    max_shape = np.array([im.shape for im in ims]).max(axis=0)\n",
    "    num_images = len(ims)\n",
    "    blob = np.zeros((num_images, max_shape[0], max_shape[1], 3),dtype=np.float32)\n",
    "    for i in range(num_images):\n",
    "        im = ims[i]\n",
    "        blob[i, 0:im.shape[0], 0:im.shape[1], :] = im\n",
    "    return blob\n",
    "\n",
    "    def prep_im_for_blob(im, pixel_means, target_size, max_size):\n",
    "        \"\"\" mean subtact and scale an image for use in a blob\"\"\"\n",
    "        im = im.astype(np.float32, copy=False)\n",
    "        im -= pixel_means\n",
    "        im_shape = im.shape\n",
    "        im_size_min = np.min(im_shape[0:2])\n",
    "        im_size_max = np.max(im_shape[0:2])\n",
    "        im_scale = float(target_size) / float(im_size_min)\n",
    "        # Prevent the biggest axis from being more than MAX_SIZE\n",
    "        if np.round(im_scale * im_size_max) > max_size:\n",
    "            im_scale = float(max_size) / float(im_size_max)\n",
    "        im = cv2.resize(im, None, None, fx=im_scale, fy=im_scale,\n",
    "                        interpolation=cv2.INTER_LINEAR)\n",
    "        return im, im_scale"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### mini-batch blobs for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "SCALES = (600,)\n",
    "pixel_means = np.array([[[102.9801, 115.9465, 122.7717]]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_minibatch(roidb,num_classes):\n",
    "    \"\"\"Given a roidb, construct a minibatch sampled from it\"\"\"\n",
    "    num_images = len(roidb)\n",
    "    # Sample random scales to use for each image in this batch\n",
    "    random_scale_inds = npr.randint(0,high=len(SCALES),size=num_images)\n",
    "    assert(128%num_images == 0), 'num_images ({}) must divide 128'.format(num_images)\n",
    "\n",
    "    # Get the input image blob, formatted for caffe\n",
    "    im_blob, im_scale = _get_image_blob(roidb, random_scale_inds)\n",
    "    blobs = {'data':im_blob}\n",
    "    assert(len(im_scale) == 1), 'Single batch only'\n",
    "    assert(len(roidb) == 1), 'Single batch only'\n",
    "\n",
    "    if USE_ALL_GT:\n",
    "        gt_inds = np.where(roidb[0]['gt_classes'] != 0)[0]\n",
    "    else:\n",
    "        gt_inds = np.where(roidb[0]['gt_classes'] != 0 & np.all(roidb[0]['gt_overlaps'].toarray() > -1.0, axis=1))[0]\n",
    "    gt_boxes = np.empty((len(gt_inds), 5), dtype=np.float32)\n",
    "    gt_boxes[:, 0:4] = roidb[0]['boxes'][gt_inds, :] * im_scale[0]\n",
    "    gt_boxes[:, 4] = roidb[0]['gt_classes'][gt_inds]\n",
    "    blobs['gt_boxes'] = gt_boxes\n",
    "    blobs['im_info'] = np.array(\n",
    "        [im_blob.shape[1], im_blob.shape[2], im_scale[0]],\n",
    "        dtype=np.float32)\n",
    "    return blobs\n",
    "\n",
    "def _get_image_blob(roidb, scale_inds):\n",
    "    \"\"\"Builds an input blob from the images in the roidb at the specified\n",
    "    scales.\n",
    "    \"\"\"\n",
    "    num_images = len(roidb)\n",
    "    processed_ims = []\n",
    "    im_scales = []\n",
    "    for i in range(num_images):\n",
    "        im = cv2.imread(roidb[i]['image'])\n",
    "        if roidb[i]['flipped']:\n",
    "            im = im[:, ::-1, :]\n",
    "        target_size = SCALES[scale_inds[i]]\n",
    "        im, im_scale = prep_im_for_blob(im, pixel_means, target_size, 1000)\n",
    "        im_scales.append(im_scale)\n",
    "        processed_ims.append(im)\n",
    "\n",
    "    # Create a blob to hold the input images\n",
    "    blob = _im_list_to_blob(processed_ims)\n",
    "    return blob, im_scales"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RoIDataLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RoIDataLayer(object):\n",
    "    \"\"\"\n",
    "    It implements a caffe Python Layer.\n",
    "    \"\"\"\n",
    "    def __init__(self, roidb, num_classes, random=False):\n",
    "        \"\"\"\n",
    "        set the roidb to be used by this layer during training\n",
    "        \"\"\"\n",
    "        self._roidb = roidb\n",
    "        self._num_classes = num_classes\n",
    "        self._random = random\n",
    "        self._shuffle_roidb_inds()\n",
    "    \n",
    "    def _shuffle_roidb_inds(self):\n",
    "        \"\"\"Randomly permute the training roidb\"\"\"\n",
    "        if self._random:\n",
    "            st0 = np.random.get_state()\n",
    "            millis = int(round(time.time()*1000)) % 4294967295\n",
    "            np.random.seed(millis)\n",
    "        self._perm = np.random.permutation(np.arange(len(self._roidb)))\n",
    "        if self._random:\n",
    "            np.random.set_state(st0)\n",
    "        \n",
    "        self._cur = 0\n",
    "    \n",
    "    def _get_next_minibatch_inds(self):\n",
    "        \"\"\"Return the roidb indices for the next minibatch.\"\"\"\n",
    "        if self._cur + 1 >= len(self._roidb):\n",
    "            self._shuffle_roidb_inds()\n",
    "        \n",
    "        db_inds = self._perm[self._cur:self._cur + 1]\n",
    "        self._cur += 1\n",
    "        return db_inds\n",
    "    \n",
    "    def _get_next_minibatch(self):\n",
    "        \"\"\"Return the blobs to be used for the next minibatch.\n",
    "        \"\"\"\n",
    "        db_inds = self._get_next_minibatch_inds()\n",
    "        minibatch_db = [self._roidb[i] for i in db_inds]\n",
    "        return get_minibatch(minibatch_db, self._num_classes)\n",
    "    \n",
    "    def forward(self):\n",
    "        \"\"\"Get blobs and copy them into this layer's top blob vector.\"\"\"\n",
    "        blobs = self._get_next_minibatch()\n",
    "        return blobs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Timer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "class Timer(object):\n",
    "    def __init__(self):\n",
    "        self.total_time = 0.\n",
    "        self.calls = 0\n",
    "        self.start_time = 0.\n",
    "        self.diff = 0.\n",
    "        self.average_time = 0.\n",
    "    def tic(self):\n",
    "        self.start_time = time.time()\n",
    "    \n",
    "    def toc(self, average=True):\n",
    "        self.diff = time.time() - self.start_time\n",
    "        self.total_time += self.diff\n",
    "        self.calls += 1\n",
    "        self.average_time = self.total_time / self.calls\n",
    "        if average:\n",
    "            return self.average_time\n",
    "        else:\n",
    "            return self.diff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### trainval - SolveWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python import pywrap_tensorflow\n",
    "import sys\n",
    "import os\n",
    "import glob\n",
    "import time\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SolveWrapper(object):\n",
    "    \"\"\"\n",
    "    A wrapper class for the training purpose\n",
    "    \"\"\"\n",
    "    def __init__(self,sess,network,imdb,roidb,valroidb,output_dir,tbdir,pretrained_model=None):\n",
    "        self.net = network\n",
    "        self.imdb = imdb\n",
    "        self.roidb = roidb\n",
    "        self.valroidb = valroidb\n",
    "        self.output_dir = output_dir\n",
    "        self.tbdir = tbdir\n",
    "        self.tbvaldir = tbdir + '_val'\n",
    "        if not os.path.exists(self.tbvaldir):\n",
    "            os.makedirs(self.tbvaldir)\n",
    "        self.pretrained_model = pretrained_model\n",
    "\n",
    "    def filter_roidb(roidb):\n",
    "        \"\"\"Remove roidb entries that have no usable RoIs\"\"\"\n",
    "        def is_valid(entry):\n",
    "            overlaps = entry['max_overlaps']\n",
    "            fg_inds = np.where(overlaps>=0.5)[0]\n",
    "            bg_inds = np.where((overlaps<0.5) & (overlaps>0.1))[0]\n",
    "            valid = len(fg_inds)>0 or len(bg_inds)>0\n",
    "            return valid\n",
    "        \n",
    "        num = len(roidb)\n",
    "        filtered_roidb = [entry for entry in roidb if is_valid(entry)]\n",
    "        num_after = len(filtered_roidb)\n",
    "        print('Filtered {} roidb entries: {} -> {}'.format(num-num_after,num,num_after))\n",
    "        return filtered_roidb\n",
    "\n",
    "    def snapshot(self,sess, iter):\n",
    "        net = self.net\n",
    "\n",
    "        if not os.path.exists(self.output_dir):\n",
    "            os.makedirs(self.output_dir)\n",
    "\n",
    "        # Store the model snapshot\n",
    "        filename = 'VGG16_faster_rcnn_iter_{:d}'.format(iter) + '.ckpt'\n",
    "        filename = os.path.join(self.output_dir, filename)\n",
    "        self.saver.save(sess, filename)\n",
    "        print('Wrote snapshot to: {:s}'.format(filename))\n",
    "\n",
    "        # Also store some meta information, random state, etc.\n",
    "        nfilename = 'VGG16_faster_rcnn_iter_{:d}'.format(iter) + '.pkl'\n",
    "        nfilename = os.path.join(self.output_dir, nfilename)\n",
    "        # current state of numpy random\n",
    "        st0 = np.random.get_state()\n",
    "        # current position in the database\n",
    "        cur = self.data_layer._cur\n",
    "        # current shuffled indexes of the database\n",
    "        perm = self.data_layer._perm\n",
    "        # current position in the validation database\n",
    "        cur_val = self.data_layer_val._cur\n",
    "        # current shuffled indexes of the validation database\n",
    "        perm_val = self.data_layer_val._perm\n",
    "\n",
    "        # Dump the meta info\n",
    "        with open(nfilename, 'wb') as fid:\n",
    "            pickle.dump(st0, fid, pickle.HIGHEST_PROTOCOL)\n",
    "            pickle.dump(cur, fid, pickle.HIGHEST_PROTOCOL)\n",
    "            pickle.dump(perm, fid, pickle.HIGHEST_PROTOCOL)\n",
    "            pickle.dump(cur_val, fid, pickle.HIGHEST_PROTOCOL)\n",
    "            pickle.dump(perm_val, fid, pickle.HIGHEST_PROTOCOL)\n",
    "            pickle.dump(iter, fid, pickle.HIGHEST_PROTOCOL)\n",
    "        \n",
    "        return filename, nfilename\n",
    "\n",
    "    def construct_graph(self,sess):\n",
    "        with sess.graph.as_default():\n",
    "            # Set the random seed for tensorflow\n",
    "            tf.keras.utils.set_random_seed(3)\n",
    "            # Build the main computation graph\n",
    "            layers = self.net_create_architecture(sess, 'TRAIN', self.imdb.num_classes, tag='default',anchor_scales = [8, 16, 32], anchor_ratios = [0.5, 1, 2])\n",
    "            # Define the loss\n",
    "            loss = layers['total_loss']\n",
    "            lr = tf.Variable(0.001, trainable=False)\n",
    "            self.optimizer = tf.compat.v1.train.MomentumOptimizer(lr=lr, momentum=0.9)\n",
    "\n",
    "            # Compute the gradients with regard to the loss\n",
    "            gvs = self.optimizer.compute_gradients(loss)\n",
    "            final_gvs = []\n",
    "            with tf.name_scope('Gradient_Mult') as scope:\n",
    "                for grad, var in gvs:\n",
    "                    scale=1\n",
    "                    if '/biases' in var.name:\n",
    "                        scale *=2\n",
    "                    if not np.allclose(scale, 1.0):\n",
    "                        grad = tf.multiply(grad, scale)\n",
    "                    final_gvs.append((grad, var))\n",
    "            train_op = self.optimizer.apply_gradients(final_gvs)\n",
    "\n",
    "            # we will handle the snapshots ourselves\n",
    "            self.saver = tf.compat.v1.train.Saver(max_to_keep=100000)\n",
    "            # Write the train and validation information to tensorboard\n",
    "            self.writer = tf.compat.v1.summary.FileWriter(self.tbdir, sess.graph)\n",
    "            self.valwriter = tf.compat.v1.summary.FileWriter(self.tbvaldir)\n",
    "        \n",
    "        return lr, train_op\n",
    "\n",
    "    def get_variables_in_checkpoint_file(self, file_name):\n",
    "        try:\n",
    "            reader = pywrap_tensorflow.NewCheckpointReader(file_name)\n",
    "            var_to_shape_map = reader.get_variable_to_shape_map()\n",
    "            return var_to_shape_map\n",
    "        except Exception as e:\n",
    "            print(str(e))\n",
    "            if \"corrupted compressed block contents\" in str(e):\n",
    "                print(\"It's likely that your checkpoint file has been compressed \"\n",
    "                      \"with SNAPPY.\")\n",
    "\n",
    "    \n",
    "    def find_previous(self):\n",
    "        sfiles = os.path.join(self.output_dir, 'vgg16_faster_rcnn'+ '_iter_*.ckpt.meta')\n",
    "        sfiles = glob.glob(sfiles)\n",
    "        sfiles.sort(key=os.path.getmtime)\n",
    "        # Get the snapshot name in TensorFlow\n",
    "        redfiles = []\n",
    "        for stepsize in [30000]:\n",
    "            redfiles.append(os.path.join(self.output_dir, 'vgg16_faster_rcnn_' + '_iter_{:d}.ckpt.meta'.format(stepsize+1)))\n",
    "        sfiles = [ss.replace('.meta', '') for ss in sfiles if ss not in redfiles]\n",
    "\n",
    "        nfiles = os.path.join(self.output_dir, 'vgg16_faster_rcnn' + '_iter_*.pkl')\n",
    "        nfiles = glob.glob(nfiles)\n",
    "        nfiles.sort(key=os.path.getmtime)\n",
    "        redfiles = [redfile.replace('.ckpt.meta', '.pkl') for redfile in redfiles]\n",
    "        nfiles = [nn for nn in nfiles if nn not in redfiles]\n",
    "\n",
    "        lsf = len(sfiles)\n",
    "        assert len(nfiles) == lsf\n",
    "\n",
    "        return lsf, nfiles, sfiles\n",
    "\n",
    "    def from_snapshot(self, sess, sfile, nfile):\n",
    "        print('Restoring model snapshots from {:s}'.format(sfile))\n",
    "        self.saver.restore(sess, sfile)\n",
    "        print('Restored.')\n",
    "        with open(nfile, 'rb') as fid:\n",
    "            st0 = pickle.load(fid)\n",
    "            cur = pickle.load(fid)\n",
    "            perm = pickle.load(fid)\n",
    "            cur_val = pickle.load(fid)\n",
    "            perm_val = pickle.load(fid)\n",
    "            last_snapshot_iter = pickle.load(fid)\n",
    "\n",
    "            np.random.set_state(st0)\n",
    "            self.data_layer._cur = cur\n",
    "            self.data_layer._perm = perm\n",
    "            self.data_layer_val._cur = cur_val\n",
    "            self.data_layer_val._perm = perm_val\n",
    "\n",
    "        return last_snapshot_iter \n",
    "\n",
    "    def initialize(self,sess):\n",
    "        np_paths = []\n",
    "        ss_paths = []\n",
    "\n",
    "        # Initialize the variables or restore them from the snapshot\n",
    "        print('Loading initial model weights from {:s}'.format(self.pretrained_model))\n",
    "        variables = tf.compat.v1.global_variables()\n",
    "        # Initialize all variables first\n",
    "        sess.run(tf.compat.v1.variables_initializer(variables, name='init'))\n",
    "        var_keep_dic = self.get_variables_in_checkpoint_file(self.pretrained_model)\n",
    "        # Get the variables to restore, ignorizing the variables to fix\n",
    "        variables_to_restore = self.net.get_variables_to_restore(variables, var_keep_dic)\n",
    "\n",
    "        restorer = tf.compat.v1.train.Saver(variables_to_restore)\n",
    "        restorer.restore(sess, self.pretrained_model)\n",
    "        print('Loaded.')\n",
    "        # Need to fix the variables before loading, so that the RGB weights are changed to BGR\n",
    "        # For VGG16 it also changes the convolutional weights fc6 and fc7 to\n",
    "        # fully connected weights\n",
    "        self.net.fix_variables(sess, self.pretrained_model)\n",
    "        print('Fixed.')\n",
    "        last_snapshot_iter = 0\n",
    "        rate = 0.001\n",
    "        stepsizes = 30000\n",
    "\n",
    "        return rate, last_snapshot_iter, stepsizes, np_paths, ss_paths\n",
    "    \n",
    "    def restore(self, sess, sfile, nfile):\n",
    "        np_path = [nfile]\n",
    "        ss_path = [sfile]\n",
    "        # Restore model from snapshots\n",
    "        last_snapshot_iter = self.from_snapshot(sess, sfile, nfile)\n",
    "        # Set the learning rate\n",
    "        rate = 0.001\n",
    "        stepsize=[]\n",
    "        for stepsize in [30000]:\n",
    "            if last_snapshot_iter > stepsize:\n",
    "                rate *= 0.1\n",
    "            else:\n",
    "                stepsizes.append(stepsize)\n",
    "\n",
    "    def remove_snapshot(self, np_paths, ss_paths):\n",
    "        to_remove = len(np_paths) - 5\n",
    "        for c in range(to_remove):\n",
    "            nfile = np_paths[0]\n",
    "            os.remove(str(nfile))\n",
    "            np_paths.remove(nfile)\n",
    "        \n",
    "        to_remove = len(ss_paths) - 5\n",
    "        for c in range(to_remove):\n",
    "            sfile = ss_paths[0]\n",
    "            # To make the code compatible to earlier versions of Tensorflow,\n",
    "            # where the naming tradition for checkpoints are different\n",
    "            if os.path.exists(str(sfile)):\n",
    "                os.remove(str(sfile))\n",
    "            else:\n",
    "                os.remove(str(sfile + '.data-00000-of-00001'))\n",
    "                os.remove(str(sfile + '.index'))\n",
    "            sfile_meta = sfile + '.meta'\n",
    "            os.remove(str(sfile_meta))\n",
    "            ss_paths.remove(sfile)\n",
    "\n",
    "    def get_training_roidb(imdb):\n",
    "        \"\"\"Returns a roidb (Region of Interest database) for use in training.\"\"\"\n",
    "        print('Appending horizontally-flipped training examples...')\n",
    "        imdb.append_flipped_images()\n",
    "        print('done')\n",
    "        print('Preparing training data...')\n",
    "        rdl_roidb.prepare_roidb(imdb)\n",
    "        print('done')\n",
    "        return imdb.roidb\n",
    "\n",
    "    def train_model(self, sess, max_iters):\n",
    "        # Build data layers for both training and validation set\n",
    "        self.data_layer = RoIDataLayer(self.roidb, self.imdb.num_classes)\n",
    "        self.data_layer_val = RoIDataLayer(self.valroidb, self.imdb.num_classes, random=True)\n",
    "\n",
    "        # Construct the computation graph\n",
    "        lr, train_op = self.construct_graph()\n",
    "\n",
    "        # Find previous snapshots if there is any to restore from\n",
    "        lsf, nfiles, sfiles = self.find_previous()\n",
    "\n",
    "        # Initialize the variables or restore them from the last snapshot\n",
    "        if lsf == 0:\n",
    "            rate, last_snapshot_iter, stepsizes, np_path, ss_paths = self.initialize(sess);\n",
    "        else:\n",
    "            rate, last_snapshot_iter, stepsizes, np_path, ss_paths = self.restore(sess, str(lsf-1))\n",
    "        timer = Timer();\n",
    "        iter = last_snapshot_iter + 1\n",
    "        last_summary_time = time.time()\n",
    "\n",
    "        stepsizes.append(max_iters)\n",
    "        stepsizes.reverse()\n",
    "        next_stepsize = stepsizes.pop()\n",
    "        while iters < max_iters+1:\n",
    "            if iter == next_stepsize_1:\n",
    "                self.snapshot(sess, iter)\n",
    "                rate *= 0.1\n",
    "                sess.run(tf.assign(lr, rate))\n",
    "                next_stepsize = stepsizes.pop()\n",
    "            \n",
    "            timer.tic()\n",
    "            blobs = self.data_layer.forward()\n",
    "\n",
    "            now = time.time()\n",
    "            if iter == 1 or now-last_summary_time > 180:\n",
    "                rpn_loss_cls, rpn_loss_box, loss_cls, loss_box, total_loss, summary = self.net.train_step_with_summary(sess, blobs, train_op)\n",
    "                self.writer.add_summary(summary, float(iter))\n",
    "                # Also check the summary on the validation set\n",
    "                blobs_val = self.data_layer_val.forward()\n",
    "                summary_val = self.net.get_summary(sess, blobs_val)\n",
    "                self.valwriter.add_summary(summary_val, float(iter))\n",
    "                last_summary_time = now\n",
    "            else:\n",
    "                rpn_loss_cls, rpn_loss_box, loss_cls, loss_box, total_loss = self.net.train_step(sess, blobs, train_op)\n",
    "            timer.toc()\n",
    "\n",
    "            # Display training information\n",
    "            if iter % 10 == 0:\n",
    "                print('iter: %d / %d, total loss: %.6f\\n >>> rpn_loss_cls: %.6f\\n '\n",
    "                      '>>> rpn_loss_box: %.6f\\n >>> loss_cls: %.6f\\n >>> loss_box: %.6f\\n >>> lr: %f' % \\\n",
    "                      (iter, max_iters, total_loss, rpn_loss_cls, rpn_loss_box, loss_cls, loss_box, lr.eval()))\n",
    "                print('speed: {:.3f}s / iter'.format(timer.average_time))\n",
    "            \n",
    "            # Snapshotting\n",
    "            if iter % 5000 == 0:\n",
    "                last_snapshot_iter = iter\n",
    "                ss_path , np_path = self.snapshot(sess, iter)\n",
    "                np_paths.append(np_path)\n",
    "                ss_paths.append(ss_path)\n",
    "\n",
    "                if len(np_paths) > 5:\n",
    "                    self.remove_snapshot(np_paths, ss_paths)\n",
    "            \n",
    "            iter += 1\n",
    "\n",
    "            if last_snapshot_iter != iter-1:\n",
    "                self.snapshot(sess,iter-1)\n",
    "            \n",
    "            self.writer.close()\n",
    "            self.valwriter.close()\n",
    "\n",
    "\n",
    "    def train_net(network, imdb, roidb, valroidb, output_dir, tb_dir, pretrained_model=None, max_iters=40000):\n",
    "        roidb = filter_roidb(roidb)\n",
    "        valroidb = filter_roidb(valroidb)\n",
    "\n",
    "        tfconfig = tf.compat.v1.ConfigProto(allow_soft_placement=True)\n",
    "        tfconfig.gpu_options.allow_growth = True\n",
    "\n",
    "        with tf.compat.v1.Session(config=tfconfig) as sess:\n",
    "            sw = SolveWrapper(sess,network,imdb,roidb,valroidb,output_dir,tb_dir,pretrained_model)\n",
    "            print('Solving')\n",
    "            sw.train_model(sess,max_iters)\n",
    "            print('done solving')\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IMDB Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import PIL\n",
    "import numpy as np\n",
    "import scipy.sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "class imdbData(object):\n",
    "    \"\"\"Image Databas\"\"\"\n",
    "\n",
    "    def __init__(self,name,classes=None):\n",
    "        self._name = name\n",
    "        self._num_classes=0\n",
    "        if not classes:\n",
    "            self._classes = []\n",
    "        else:\n",
    "            self._classes = classes\n",
    "        self._image_index = []\n",
    "        self._obj_proposer= 'gt'\n",
    "        self._roidb = None\n",
    "        self._roidb_handler = self.default_roidb\n",
    "        # Use this dict for storing dataset specific config options\n",
    "        self.config = {}\n",
    "\n",
    "    @property\n",
    "    def name(self):\n",
    "        return self._name\n",
    "    \n",
    "    @property\n",
    "    def num_classes(self):\n",
    "        return len(self._classes)\n",
    "    \n",
    "    @property\n",
    "    def classes(self):\n",
    "        return self._classes\n",
    "\n",
    "    @property\n",
    "    def image_index(self):\n",
    "        return self._image_index\n",
    "\n",
    "    @property\n",
    "    def roidb_handler(self):\n",
    "        return self._roidb_handler\n",
    "\n",
    "    @roidb_handler.setter\n",
    "    def roidb_handler(self,val):\n",
    "        self._roidb_handler = val\n",
    "    \n",
    "    def set_proposal_method(self,method):\n",
    "        method = eval('self.'+method+'_roidb')\n",
    "        self.roidb_handler = method\n",
    "    \n",
    "    @property\n",
    "    def roidb(self):\n",
    "        # A roidb is a list of dictionaries, each with the following keys:\n",
    "        #   boxes\n",
    "        #   gt_overlaps\n",
    "        #   gt_classes\n",
    "        #   flipped\n",
    "        if self._roidb is not None:\n",
    "            return self._roidb\n",
    "        self._roidb = self.roidb_handler()\n",
    "        return self._roidb\n",
    "\n",
    "    @property\n",
    "    def cache_path(self):\n",
    "        cache_path = os.path.abspath(os.path.join(r'C:\\Users\\kc510\\Documents\\Projects\\Projects_MLOps\\Project_Faster_RCNN\\data', 'cache'))\n",
    "        if not os.path.exists(cache_path):\n",
    "            os.makedirs(cache_path)\n",
    "        return cache_path\n",
    "    \n",
    "    @property\n",
    "    def num_images(self):\n",
    "        return len(self.image_index)\n",
    "\n",
    "    def image_path_at(self, i):\n",
    "        raise NotImplementedError\n",
    "    \n",
    "    def default_roidb(self):\n",
    "        raise NotImplementedError\n",
    "    \n",
    "    def evaluate_detections(self, all_boxes, output_dir=None):\n",
    "        \"\"\"\n",
    "        all_boxes is a list of length number-of-classes.\n",
    "        Each list element is a list of length number-of-images.\n",
    "        Each of those list elements is either an empty list []\n",
    "        or a numpy array of detection.\n",
    "        all_boxes[class][image] = [] or np.array of shape #dets x 5\n",
    "        \"\"\"\n",
    "        raise NotImplementedError\n",
    "    \n",
    "    def _get_width(self):\n",
    "        return [PIL.Image.open(self.image_path_at(i)).size[0] for i in range(self.num_images)]\n",
    "    \n",
    "    def append_flipped_images(self):\n",
    "        num_images = self.num_images\n",
    "        widths = self._get_width()\n",
    "        for i in range(num_images):\n",
    "            boxes = self.roidb[i]['boxes'].copy()\n",
    "            oldx1 = boxes[:, 0].copy()\n",
    "            oldx2 = boxes[:, 2].copy()\n",
    "            boxes[:, 0] = widths[i] - oldx2 - 1\n",
    "            boxes[:, 2] = widths[i] - oldx1 - 1\n",
    "            assert (boxes[:, 2] >= boxes[:, 0]).all()\n",
    "            entry = {'boxes': boxes,\n",
    "                     'gt_overlaps': self.roidb[i]['gt_overlaps'],\n",
    "                     'gt_classes': self.roidb[i]['gt_classes'],\n",
    "                     'flipped': True}\n",
    "            self.roidb.append(entry)\n",
    "        self._image_index = self._image_index * 2\n",
    "    \n",
    "    def evaluate_recall(self, candidate_boxes=None, thresholds=None, area='all', limit=None):\n",
    "        \"\"\" Evaluate detection proposal recall metrics \n",
    "        Return:\n",
    "            results: dictionary of results with keys\n",
    "                'ar': average recall\n",
    "                'recalls' : vector recalls at each IoU overlap threshold\n",
    "                'thresholds' : vector of IoU overlap thresholds\n",
    "                'gt_overlaps' : vector of all ground-truth overlaps\n",
    "        \"\"\"\n",
    "        # Record max overlap value for each gt box\n",
    "        # Return vector of overlap values\n",
    "        areas = {'all': 0, 'small': 1, 'medium': 2, 'large': 3, '96-128': 4, '128-256': 5, '256-512': 6, '512-inf': 7}\n",
    "        area_ranges = [[0**2, 1e5**2], [0**2, 32**2], [32**2, 96**2], [96**2, 128**2], [128**2, 256**2], [256**2, 512**2], [512**2, 1e5**2]]\n",
    "        assert area in areas, 'unknown area range: {}'.format(area)\n",
    "        area_range = area_ranges[areas[area]]\n",
    "        gt_overlaps = np.zeros(0)\n",
    "        num_pos = 0\n",
    "        for i in range(self.num_images):\n",
    "            # Checking for max_overlaps == 1 avoids including crowd annotations\n",
    "            # (...pretty hacking :/)\n",
    "            max_gt_overlaps = self.roidb[i]['gt_overlaps'].toarray().max(axis=1)\n",
    "            gt_inds = np.where((self.roidb[i]['gt_classes'] > 0) & (max_gt_overlaps == 1))[0]\n",
    "            gt_boxes = self.roidb[i]['boxes'][gt_inds, :]\n",
    "            gt_areas = self.roidb[i]['seg_areas'][gt_inds]\n",
    "            valid_gt_inds = np.where((gt_areas >= area_range[0]) & (gt_areas <= area_range[1]))[0]\n",
    "            gt_boxes = gt_boxes[valid_gt_inds, :]\n",
    "            num_pos += len(valid_gt_inds)\n",
    "            \n",
    "            if candidate_boxes is None:\n",
    "                # If candidate_boxes is not supplied, the default is to use the\n",
    "                # non-ground-truth boxes from this roidb\n",
    "                non_gt_inds = np.where(self.roidb[i]['gt_classes'] == 0)[0]\n",
    "                boxes = self.roidb[i]['boxes'][non_gt_inds, :]\n",
    "            else:\n",
    "                boxes = candidate_boxes[i]\n",
    "            if boxes.shape[0] == 0:\n",
    "                continue\n",
    "            if limit is not None and boxes.shape[0] > limit:\n",
    "                boxes = boxes[:limit, :]\n",
    "            \n",
    "            overlaps = bbox_overlaps(boxes.astype(np.float), gt_boxes.astype(np.float))\n",
    "\n",
    "            _gt_overlaps = np.zeros((gt_boxes.shape[0]))\n",
    "            for j in range(gt_boxes.shape[0]):\n",
    "                # find which proposal box maximally covers each gt box\n",
    "                argmax_overlaps = overlaps.argmax(axis=0)\n",
    "                # and get the iou amount of coverage for each gt box\n",
    "                max_overlaps = overlaps.max(axis=0)\n",
    "                # find which gt box is 'best' covered (i.e. 'best' = most iou)\n",
    "                gt_ind = max_overlaps.argmax()\n",
    "                gt_ovr = max_overlaps.max()\n",
    "                assert(gt_ovr >= 0)\n",
    "                # find the proposal box that covers the best covered gt box\n",
    "                box_ind = argmax_overlaps[gt_ind]\n",
    "                # record the iou coverage of this gt box\n",
    "                _gt_overlaps[j] = overlaps[box_ind, gt_ind]\n",
    "                assert(_gt_overlaps[j] == gt_ovr)\n",
    "                # mark the proposal box and the gt box as used\n",
    "                overlaps[box_ind, :] = -1\n",
    "                overlaps[:, gt_ind] = -1\n",
    "            # append recorded iou coverage level\n",
    "            gt_overlaps = np.hstack((gt_overlaps, _gt_overlaps))\n",
    "\n",
    "        gt_overlaps = np.sort(gt_overlaps)\n",
    "        if thresholds is None:\n",
    "            step = 0.05\n",
    "            thresholds = np.arange(0.5, 0.95 + 1e-5, step)\n",
    "        recalls = np.zeros_like(thresholds)\n",
    "        # compute recall for each iou threshold\n",
    "        for i, t in enumerate(thresholds):\n",
    "            recalls[i] = (gt_overlaps >= t).sum() / float(num_pos)\n",
    "        # ar = 2 * np.trapz(recalls, thresholds)\n",
    "        ar = recalls.mean()\n",
    "        return {'ar': ar, 'recalls': recalls, 'thresholds': thresholds, 'gt_overlaps': gt_overlaps}\n",
    "\n",
    "    def create_roidb_from_box_list(self,box_list,gt_roidb):\n",
    "        assert len(box_list) == self.num_images, 'Number of boxes must match number of ground-truth images'\n",
    "        roidb = []\n",
    "        for i in range(self.num_images):\n",
    "            boxes = box_list[i]\n",
    "            num_boxes = boxes.shape[0]\n",
    "            overlaps = np.zeros((num_boxes,self.num_classes),dtype=np.float32)\n",
    "\n",
    "            if gt_roidb is not None and gt_roidb[i]['boxes'].size > 0:\n",
    "                gt_boxes = gt_roidb[i]['boxes']\n",
    "                gt_classes = gt_roidb[i]['gt_classes']\n",
    "                gt_overlaps = bbox_overlaps(boxes.astype(np.float),gt_boxes.astype(np.float))\n",
    "                argmaxes = gt_overlaps.argmax(axis=1)\n",
    "                maxes = gt_overlaps.max(axis=1)\n",
    "                I = np.where(maxes>0)[0]\n",
    "                overlaps[I,gt_classes[argmaxes[I]]] = maxes[I]\n",
    "            \n",
    "            overlaps = scipy.sparse.csr_matrix(overlaps)\n",
    "            roidb.append({'boxes':boxes,'gt_classes':np.zeros((num_boxes,),dtype=np.int32),'gt_overlaps':overlaps,'flipped':False})\n",
    "        return roidb\n",
    "\n",
    "    @staticmethod\n",
    "    def merge_roidbs(a,b):\n",
    "        assert len(a) == len(b)\n",
    "        for i in range(len(a)):\n",
    "            a[i]['boxes'] = np.vstack((a[i]['boxes'],b[i]['boxes']))\n",
    "            a[i]['gt_classes'] = np.hstack((a[i]['gt_classes'],b[i]['gt_classes']))\n",
    "            a[i]['gt_overlaps'] = scipy.sparse.vstack([a[i]['gt_overlaps'],b[i]['gt_overlaps']])\n",
    "        return a\n",
    "    \n",
    "    def competition_mode(self, on):\n",
    "        \"\"\"Turn competition mode on or off.\"\"\"\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ds_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unique_boxes(boxes, scale=1.0):\n",
    "    \"\"\" return indices of unique boxes \"\"\"\n",
    "    v = np.array([1, 1e3, 1e6, 1e9])\n",
    "    hashes = np.round(boxes*scale).dot(v)\n",
    "    _, index = np.unique(hashes, return_index=True)\n",
    "    return np.sort(index)\n",
    "\n",
    "def xywh_to_xyxy(boxes):\n",
    "    \"\"\" Convert [x y w h] box format to [x1 y1 x2 y2] format \"\"\"\n",
    "    return np.hstack((boxes[:, 0:2], boxes[:, 0:2] + boxes[:, 2:4] - 1))\n",
    "\n",
    "def xyxy_to_xywh(boxes):\n",
    "    \"\"\" Convert [x1 y1 x2 y2] box format to [x y w h] format \"\"\"\n",
    "    return np.hstack((boxes[:, 0:2], boxes[:, 2:4] - boxes[:, 0:2] + 1))\n",
    "\n",
    "def validate_boxes(boxes,width=0,height=0):\n",
    "    \"\"\"Check that a set of boxes is valid.\"\"\"\n",
    "    x1 = boxes[:, 0]\n",
    "    y1 = boxes[:, 1]\n",
    "    x2 = boxes[:, 2]\n",
    "    y2 = boxes[:, 3]\n",
    "    assert (x1 >= 0).all()\n",
    "    assert (y1 >= 0).all()\n",
    "    assert (x2 >= x1).all()\n",
    "    assert (y2 >= y1).all()\n",
    "    assert (x2 < width).all()\n",
    "    assert (y2 < height).all()\n",
    "\n",
    "def filter_small_boxes(boxes,min_size):\n",
    "    w = boxes[:, 2] - boxes[:, 0]\n",
    "    h = boxes[:, 3] - boxes[:, 1]\n",
    "    keep = np.where((w >= min_size) & (h > min_size))[0]\n",
    "    return keep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### COCO Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import uuid\n",
    "from pycocotools.coco import COCO\n",
    "from pycocotools import mask as COCOmask\n",
    "from pycocotools.cocoeval import COCOeval\n",
    "import os\n",
    "import os.path as osp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "class coco(imdbData):\n",
    "      def __init__(self, image_set, year):\n",
    "    print(\"image set : \",image_set)\n",
    "    print(\"year : \",year)\n",
    "    print(\"type of image set: \", type(image_set))\n",
    "    print(\"type of year: \",type(year))\n",
    "    imdbData.__init__(self, 'coco_' + year + '_' + image_set)\n",
    "    # COCO specific config options\n",
    "    self.config = {'use_salt': True,\n",
    "                   'cleanup': True}\n",
    "    # name, paths\n",
    "    self._year = year\n",
    "    self._image_set = image_set\n",
    "    self._data_path = osp.join(r'FasterRCNN\\data', 'coco')\n",
    "    # load COCO API, classes, class <-> id mappings\n",
    "    self._COCO = COCO(self._get_ann_file())\n",
    "    cats = self._COCO.loadCats(self._COCO.getCatIds())\n",
    "    self._classes = tuple(['__background__'] + [c['name'] for c in cats])\n",
    "    self._class_to_ind = dict(list(zip(self.classes, list(range(self.num_classes)))))\n",
    "    self._class_to_coco_cat_id = dict(list(zip([c['name'] for c in cats],\n",
    "                                               self._COCO.getCatIds())))\n",
    "    self._image_index = self._load_image_set_index()\n",
    "    # Default to roidb handler\n",
    "    self.set_proposal_method('gt')\n",
    "    self.competition_mode(False)\n",
    "\n",
    "    # Some image sets are \"views\" (i.e. subsets) into others.\n",
    "    # For example, minival2014 is a random 5000 image subset of val2014.\n",
    "    # This mapping tells us where the view's images and proposals come from.\n",
    "    self._view_map = {\n",
    "      'minival2014': 'val2014',  # 5k val2014 subset\n",
    "      'valminusminival2014': 'val2014',  # val2014 \\setminus minival2014\n",
    "      'test-dev2015': 'test2015',\n",
    "    }\n",
    "    coco_name = image_set + year  # e.g., \"val2014\"\n",
    "    self._data_name = (self._view_map[coco_name]\n",
    "                       if coco_name in self._view_map\n",
    "                       else coco_name)\n",
    "    # Dataset splits that have ground-truth annotations (test splits\n",
    "    # do not have gt annotations)\n",
    "    self._gt_splits = ('train', 'val', 'minival')\n",
    "\n",
    "  def _get_ann_file(self):\n",
    "    prefix = 'instances' if self._image_set.find('test') == -1 \\\n",
    "      else 'image_info'\n",
    "    return osp.join(self._data_path, 'annotations',\n",
    "                    prefix + '_' + self._image_set + self._year + '.json')\n",
    "\n",
    "  def _load_image_set_index(self):\n",
    "    \"\"\"\n",
    "    Load image ids.\n",
    "    \"\"\"\n",
    "    image_ids = self._COCO.getImgIds()\n",
    "    return image_ids\n",
    "\n",
    "  def _get_widths(self):\n",
    "    anns = self._COCO.loadImgs(self._image_index)\n",
    "    widths = [ann['width'] for ann in anns]\n",
    "    return widths\n",
    "\n",
    "  def image_path_at(self, i):\n",
    "    \"\"\"\n",
    "    Return the absolute path to image i in the image sequence.\n",
    "    \"\"\"\n",
    "    return self.image_path_from_index(self._image_index[i])\n",
    "\n",
    "  def image_path_from_index(self, index):\n",
    "    \"\"\"\n",
    "    Construct an image path from the image's \"index\" identifier.\n",
    "    \"\"\"\n",
    "    # Example image path for index=119993:\n",
    "    #   images/train2014/COCO_train2014_000000119993.jpg\n",
    "    file_name = ('COCO_' + self._data_name + '_' +\n",
    "                 str(index).zfill(12) + '.jpg')\n",
    "    image_path = osp.join(r\"FasterRCNN\\data\\coco\", 'images',\n",
    "                          self._data_name, file_name)\n",
    "    assert osp.exists(image_path), \\\n",
    "      'Path does not exist: {}'.format(image_path)\n",
    "    return image_path\n",
    "\n",
    "  def gt_roidb(self):\n",
    "    \"\"\"\n",
    "    Return the database of ground-truth regions of interest.\n",
    "    This function loads/saves from/to a cache file to speed up future calls.\n",
    "    \"\"\"\n",
    "    cache_file = osp.join(self.cache_path, self.name + '_gt_roidb.pkl')\n",
    "    if osp.exists(cache_file):\n",
    "      with open(cache_file, 'rb') as fid:\n",
    "        roidb = pickle.load(fid)\n",
    "      print('{} gt roidb loaded from {}'.format(self.name, cache_file))\n",
    "      return roidb\n",
    "\n",
    "    gt_roidb = [self._load_coco_annotation(index)\n",
    "                for index in self._image_index]\n",
    "\n",
    "    with open(cache_file, 'wb') as fid:\n",
    "      pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)\n",
    "    print('wrote gt roidb to {}'.format(cache_file))\n",
    "    return gt_roidb\n",
    "\n",
    "  def _load_coco_annotation(self, index):\n",
    "    \"\"\"\n",
    "    Loads COCO bounding-box instance annotations. Crowd instances are\n",
    "    handled by marking their overlaps (with all categories) to -1. This\n",
    "    overlap value means that crowd \"instances\" are excluded from training.\n",
    "    \"\"\"\n",
    "    im_ann = self._COCO.loadImgs(index)[0]\n",
    "    width = im_ann['width']\n",
    "    height = im_ann['height']\n",
    "\n",
    "    annIds = self._COCO.getAnnIds(imgIds=index, iscrowd=None)\n",
    "    objs = self._COCO.loadAnns(annIds)\n",
    "    # Sanitize bboxes -- some are invalid\n",
    "    valid_objs = []\n",
    "    for obj in objs:\n",
    "      x1 = np.max((0, obj['bbox'][0]))\n",
    "      y1 = np.max((0, obj['bbox'][1]))\n",
    "      x2 = np.min((width - 1, x1 + np.max((0, obj['bbox'][2] - 1))))\n",
    "      y2 = np.min((height - 1, y1 + np.max((0, obj['bbox'][3] - 1))))\n",
    "      if obj['area'] > 0 and x2 >= x1 and y2 >= y1:\n",
    "        obj['clean_bbox'] = [x1, y1, x2, y2]\n",
    "        valid_objs.append(obj)\n",
    "    objs = valid_objs\n",
    "    num_objs = len(objs)\n",
    "\n",
    "    boxes = np.zeros((num_objs, 4), dtype=np.uint16)\n",
    "    gt_classes = np.zeros((num_objs), dtype=np.int32)\n",
    "    overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)\n",
    "    seg_areas = np.zeros((num_objs), dtype=np.float32)\n",
    "\n",
    "    # Lookup table to map from COCO category ids to our internal class\n",
    "    # indices\n",
    "    coco_cat_id_to_class_ind = dict([(self._class_to_coco_cat_id[cls],\n",
    "                                      self._class_to_ind[cls])\n",
    "                                     for cls in self._classes[1:]])\n",
    "\n",
    "    for ix, obj in enumerate(objs):\n",
    "      cls = coco_cat_id_to_class_ind[obj['category_id']]\n",
    "      boxes[ix, :] = obj['clean_bbox']\n",
    "      gt_classes[ix] = cls\n",
    "      seg_areas[ix] = obj['area']\n",
    "      if obj['iscrowd']:\n",
    "        # Set overlap to -1 for all classes for crowd objects\n",
    "        # so they will be excluded during training\n",
    "        overlaps[ix, :] = -1.0\n",
    "      else:\n",
    "        overlaps[ix, cls] = 1.0\n",
    "\n",
    "    validate_boxes(boxes, width=width, height=height)\n",
    "    overlaps = scipy.sparse.csr_matrix(overlaps)\n",
    "    return {'width': width,\n",
    "            'height': height,\n",
    "            'boxes': boxes,\n",
    "            'gt_classes': gt_classes,\n",
    "            'gt_overlaps': overlaps,\n",
    "            'flipped': False,\n",
    "            'seg_areas': seg_areas}\n",
    "\n",
    "  def _get_widths(self):\n",
    "    return [r['width'] for r in self.roidb]\n",
    "\n",
    "  def append_flipped_images(self):\n",
    "    num_images = self.num_images\n",
    "    widths = self._get_widths()\n",
    "    for i in range(num_images):\n",
    "      boxes = self.roidb[i]['boxes'].copy()\n",
    "      oldx1 = boxes[:, 0].copy()\n",
    "      oldx2 = boxes[:, 2].copy()\n",
    "      boxes[:, 0] = widths[i] - oldx2 - 1\n",
    "      boxes[:, 2] = widths[i] - oldx1 - 1\n",
    "      assert (boxes[:, 2] >= boxes[:, 0]).all()\n",
    "      entry = {'width': widths[i],\n",
    "               'height': self.roidb[i]['height'],\n",
    "               'boxes': boxes,\n",
    "               'gt_classes': self.roidb[i]['gt_classes'],\n",
    "               'gt_overlaps': self.roidb[i]['gt_overlaps'],\n",
    "               'flipped': True,\n",
    "               'seg_areas': self.roidb[i]['seg_areas']}\n",
    "\n",
    "      self.roidb.append(entry)\n",
    "    self._image_index = self._image_index * 2\n",
    "\n",
    "  def _get_box_file(self, index):\n",
    "    # first 14 chars / first 22 chars / all chars + .mat\n",
    "    # COCO_val2014_0/COCO_val2014_000000447/COCO_val2014_000000447991.mat\n",
    "    file_name = ('COCO_' + self._data_name +\n",
    "                 '_' + str(index).zfill(12) + '.mat')\n",
    "    return osp.join(file_name[:14], file_name[:22], file_name)\n",
    "\n",
    "  def _print_detection_eval_metrics(self, coco_eval):\n",
    "    IoU_lo_thresh = 0.5\n",
    "    IoU_hi_thresh = 0.95\n",
    "\n",
    "    def _get_thr_ind(coco_eval, thr):\n",
    "      ind = np.where((coco_eval.params.iouThrs > thr - 1e-5) &\n",
    "                     (coco_eval.params.iouThrs < thr + 1e-5))[0][0]\n",
    "      iou_thr = coco_eval.params.iouThrs[ind]\n",
    "      assert np.isclose(iou_thr, thr)\n",
    "      return ind\n",
    "\n",
    "    ind_lo = _get_thr_ind(coco_eval, IoU_lo_thresh)\n",
    "    ind_hi = _get_thr_ind(coco_eval, IoU_hi_thresh)\n",
    "    # precision has dims (iou, recall, cls, area range, max dets)\n",
    "    # area range index 0: all area ranges\n",
    "    # max dets index 2: 100 per image\n",
    "    precision = \\\n",
    "      coco_eval.eval['precision'][ind_lo:(ind_hi + 1), :, :, 0, 2]\n",
    "    ap_default = np.mean(precision[precision > -1])\n",
    "    print(('~~~~ Mean and per-category AP @ IoU=[{:.2f},{:.2f}] '\n",
    "           '~~~~').format(IoU_lo_thresh, IoU_hi_thresh))\n",
    "    print('{:.1f}'.format(100 * ap_default))\n",
    "    for cls_ind, cls in enumerate(self.classes):\n",
    "      if cls == '__background__':\n",
    "        continue\n",
    "      # minus 1 because of __background__\n",
    "      precision = coco_eval.eval['precision'][ind_lo:(ind_hi + 1), :, cls_ind - 1, 0, 2]\n",
    "      ap = np.mean(precision[precision > -1])\n",
    "      print('{:.1f}'.format(100 * ap))\n",
    "\n",
    "    print('~~~~ Summary metrics ~~~~')\n",
    "    coco_eval.summarize()\n",
    "\n",
    "  def _do_detection_eval(self, res_file, output_dir):\n",
    "    ann_type = 'bbox'\n",
    "    coco_dt = self._COCO.loadRes(res_file)\n",
    "    coco_eval = COCOeval(self._COCO, coco_dt)\n",
    "    coco_eval.params.useSegm = (ann_type == 'segm')\n",
    "    coco_eval.evaluate()\n",
    "    coco_eval.accumulate()\n",
    "    self._print_detection_eval_metrics(coco_eval)\n",
    "    eval_file = osp.join(output_dir, 'detection_results.pkl')\n",
    "    with open(eval_file, 'wb') as fid:\n",
    "      pickle.dump(coco_eval, fid, pickle.HIGHEST_PROTOCOL)\n",
    "    print('Wrote COCO eval results to: {}'.format(eval_file))\n",
    "\n",
    "  def _coco_results_one_category(self, boxes, cat_id):\n",
    "    results = []\n",
    "    for im_ind, index in enumerate(self.image_index):\n",
    "      dets = boxes[im_ind].astype(np.float)\n",
    "      if dets == []:\n",
    "        continue\n",
    "      scores = dets[:, -1]\n",
    "      xs = dets[:, 0]\n",
    "      ys = dets[:, 1]\n",
    "      ws = dets[:, 2] - xs + 1\n",
    "      hs = dets[:, 3] - ys + 1\n",
    "      results.extend(\n",
    "        [{'image_id': index,\n",
    "          'category_id': cat_id,\n",
    "          'bbox': [xs[k], ys[k], ws[k], hs[k]],\n",
    "          'score': scores[k]} for k in range(dets.shape[0])])\n",
    "    return results\n",
    "\n",
    "  def _write_coco_results_file(self, all_boxes, res_file):\n",
    "    # [{\"image_id\": 42,\n",
    "    #   \"category_id\": 18,\n",
    "    #   \"bbox\": [258.15,41.29,348.26,243.78],\n",
    "    #   \"score\": 0.236}, ...]\n",
    "    results = []\n",
    "    for cls_ind, cls in enumerate(self.classes):\n",
    "      if cls == '__background__':\n",
    "        continue\n",
    "      print('Collecting {} results ({:d}/{:d})'.format(cls, cls_ind,\n",
    "                                                       self.num_classes - 1))\n",
    "      coco_cat_id = self._class_to_coco_cat_id[cls]\n",
    "      results.extend(self._coco_results_one_category(all_boxes[cls_ind],\n",
    "                                                     coco_cat_id))\n",
    "    print('Writing results json to {}'.format(res_file))\n",
    "    with open(res_file, 'w') as fid:\n",
    "      json.dump(results, fid)\n",
    "\n",
    "  def evaluate_detections(self, all_boxes, output_dir):\n",
    "    res_file = osp.join(output_dir, ('detections_' +\n",
    "                                     self._image_set +\n",
    "                                     self._year +\n",
    "                                     '_results'))\n",
    "    if self.config['use_salt']:\n",
    "      res_file += '_{}'.format(str(uuid.uuid4()))\n",
    "    res_file += '.json'\n",
    "    self._write_coco_results_file(all_boxes, res_file)\n",
    "    # Only do evaluation on non-test sets\n",
    "    if self._image_set.find('test') == -1:\n",
    "      self._do_detection_eval(res_file, output_dir)\n",
    "    # Optionally cleanup results json file\n",
    "    if self.config['cleanup']:\n",
    "      os.remove(res_file)\n",
    "\n",
    "  def competition_mode(self, on):\n",
    "    if on:\n",
    "      self.config['use_salt'] = False\n",
    "      self.config['cleanup'] = False\n",
    "    else:\n",
    "      self.config['use_salt'] = True\n",
    "      self.config['cleanup'] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_output_dir(imdb, weights_filename):\n",
    "    \"\"\"\n",
    "    Return the directory where experimental artifacts are placed.\n",
    "    If the directory does not exist, it is created.\n",
    "    A canonical path is built using the name from an imdb and a network\n",
    "    (if not None).\n",
    "    \"\"\"\n",
    "    outdir = osp.abspath(osp.join(r'C:\\Users\\kc510\\Documents\\Projects\\Projects_MLOps\\Project_Faster_RCNN\\saved_models', 'output','default', imdb.name))\n",
    "    if weights_filename is not None:\n",
    "        outdir = osp.join(outdir, weights_filename)\n",
    "    if not os.path.exists(outdir):\n",
    "        os.makedirs(outdir)\n",
    "    return outdir\n",
    "\n",
    "def get_output_tb_dir(imdb,weights_filename):\n",
    "    \"\"\"\n",
    "    Return the directory where tensorflow summaries are placed.\n",
    "    If the directory does not exist, it is created.\n",
    "    A canonical path is built using the name from an imdb and a network\n",
    "    (if not None).\n",
    "    \"\"\"\n",
    "    outdir = osp.abspath(osp.join(r'C:\\Users\\kc510\\Documents\\Projects\\Projects_MLOps\\Project_Faster_RCNN\\saved_models', 'tensorboard', 'default', imdb.name))\n",
    "    if weights_filename is not None:\n",
    "        outdir = osp.join(outdir, weights_filename)\n",
    "    if not os.path.exists(outdir):\n",
    "        os.makedirs(outdir)\n",
    "    return outdir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset Factory\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "__sets = {}\n",
    "import numpy as np\n",
    "\n",
    "for year in ['2014']:\n",
    "    for split in ['train', 'val']:\n",
    "        name = 'coco_{}_{}'.format(year, split)\n",
    "        __sets[name] = (lambda split=split, year=year: coco(split, year))\n",
    "\n",
    "# Set up coco_2015_<split>\n",
    "for year in ['2015']:\n",
    "  for split in ['test', 'test-dev']:\n",
    "    name = 'coco_{}_{}'.format(year, split)\n",
    "    __sets[name] = (lambda split=split, year=year: coco(split, year))\n",
    "\n",
    "def get_imdb(name):\n",
    "    \"\"\"Get an imdb (image database) by name.\"\"\"\n",
    "    if name not in __sets:\n",
    "        raise KeyError('Unknown dataset: {}'.format(name))\n",
    "    return __sets[name]()\n",
    "\n",
    "\n",
    "def list_imdbs():\n",
    "    \"\"\"List all registered imdbs.\"\"\"\n",
    "    return list(__sets.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pprint\n",
    "import argparse\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=21.26s)\n",
      "creating index...\n",
      "index created!\n",
      "loading annotations into memory...\n",
      "Done (t=23.86s)\n",
      "creating index...\n",
      "index created!\n",
      "Output will be saved to `C:\\Users\\kc510\\Documents\\Projects\\Projects_MLOps\\Project_Faster_RCNN\\saved_models\\output\\default\\coco_2014_train\\default`\n",
      "TensorFlow summaries will be saved to `C:\\Users\\kc510\\Documents\\Projects\\Projects_MLOps\\Project_Faster_RCNN\\saved_models\\tensorboard\\default\\coco_2014_train\\default`\n"
     ]
    }
   ],
   "source": [
    "def combined_roidb(imdb_names):\n",
    "    \"\"\"\n",
    "    Combine multiple roidbs\n",
    "    \"\"\"\n",
    "    def get_roidb(imdb_name):\n",
    "        imdb = get_imdb(imdb_name)\n",
    "        print('Loaded dataset `{:s}` for training'.format(imdb.name))\n",
    "        imdb.set_proposal_method('gt')\n",
    "        print('Set proposal method: {:s}'.format('gt'))\n",
    "        roidb = SolveWrapper.get_training_roidb(imdb)\n",
    "        return roidb\n",
    "    \n",
    "    roidbs = [get_roidb(s) for s in imdb_names.split('+')]\n",
    "    roidb = roidbs[0]\n",
    "    if len(roidbs) > 1:\n",
    "        for r in roidbs[1:]:\n",
    "            roidb.extend(r)\n",
    "        tmp = get_imdb(imdb_names.split('+')[1])\n",
    "        print(imdb_names)\n",
    "        imdb = imdbData(imdb_names, tmp.classes)\n",
    "    else:\n",
    "        imdb = get_imdb(imdb_names)\n",
    "    return imdb, roidb\n",
    "\n",
    "np.random.seed(3)\n",
    "\n",
    "# train set\n",
    "imdb, roidb = combined_roidb('coco_2014_train')\n",
    "output_dir = get_output_dir(imdb, 'default')\n",
    "print('Output will be saved to `{:s}`'.format(output_dir))\n",
    "tb_dir = get_output_tb_dir(imdb, 'default')\n",
    "print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir))\n",
    "\n",
    "orgflip = True\n",
    "USE_FLIPPED=False\n",
    "_ , valroidb = combined_roidb('coco_2014_val')\n",
    "print('{:d} validation roidb entries'.format(len(valroidb)))\n",
    "USE_FILLPED=orgflip\n",
    "\n",
    "# load network\n",
    "net = vgg16()\n",
    "\n",
    "SolveWrapper.train_net(net, imdb, roidb, valroidb, output_dir, tb_dir, pretrained_model='/content/drive/MyDrive/FasterRCNN/data/imagenet_weights/vgg16.ckpt', max_iters=40000)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "578ce72e72bd9f13049fd4c13d9f5b1c81715c13ddea0a3c61ff70756cb5d6d4"
  },
  "kernelspec": {
   "display_name": "Python 3.9.2 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}