object_detection/ossdc_vbacc_object_detection_notebook_colaboratory.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ossdc_vbacc_object_detection_notebook_colaboratory.ipynb",
      "version": "0.3.2",
      "views": {},
      "default_view": {},
      "provenance": [
        {
          "file_id": "1B3ugwX_LrNukEaf8yZHvXYotSjjSwhKD",
          "timestamp": 1517258354611
        },
        {
          "file_id": "10hu0d9Ind2TwZXxDoeh7Yc4Qi1WZ2I24",
          "timestamp": 1516545640031
        }
      ],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "rEBnky83Robi",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Use this IPython notebook on [Google Colaboratory](https://colab.research.google.com) to reproduce the video here:\n",
        "\n",
        "> SSD Tensorflow based car detection and tracking demo for OSSDC.org VisionBasedACC PS3/PS4 simulator\n",
        "\n",
        "> https://youtu.be/dqnjHqwP68Y\n",
        "\n",
        "Code here:\n",
        "\n",
        "> https://github.com/OSSDC/OSSDC-VisionBasedACC\n",
        "\n",
        "Make sure you enable GPU in Colaboratory using menu path Runtime -> Change runtime type\n",
        "\n",
        "Join our efforts in [Open Source Self Driving Car Initiative](http://OSSDC.org)!\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "HPmOjGVBquUW",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "#install opencv and other packages\n",
        "!apt-get -qq install -y libsm6 libxext6 && pip install -q -U opencv-python && pip install -q -U youtube-dl && pip install -q -U pafy && pip install -q -U imtools"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "T5Qdi64Hqegg",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "import math\n",
        "#import random\n",
        "\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import cv2\n",
        "import pafy\n",
        "\n",
        "#from imutils.video import WebcamVideoStream\n",
        "\n",
        "slim = tf.contrib.slim\n",
        "\n",
        "%pylab inline \n",
        "from IPython.display import clear_output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8-SSYlKPqegy",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "#%matplotlib inline\n",
        "#import matplotlib.pyplot as plt\n",
        "#import matplotlib.image as mpimg\n",
        "#from skimage import io\n",
        "import time\n",
        "import subprocess\n",
        "\n",
        "precision = 10\n",
        "from datetime import datetime\n",
        "\n",
        "def getCurrentClock():\n",
        "    #return time.clock()\n",
        "    return datetime.now()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "0jv-Tg9qqeg6",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "import sys\n",
        "\n",
        "#download OSSDC-VisionBasedACC Git repository\n",
        "!rm master.zip\n",
        "!wget https://github.com/OSSDC/OSSDC-VisionBasedACC/archive/master.zip\n",
        "!rm -r OSSDC-VisionBasedACC-master\n",
        "!unzip master.zip\n",
        "\n",
        "sys.path.append('.')\n",
        "import os"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5pFtiW9R1SKR",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "!cp -r OSSDC-VisionBasedACC-master/object_detection/* .\n",
        "!ls -l checkpoints\n",
        "!rm checkpoints/*"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "QbYzlXWO1wsp",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "import requests\n",
        "\n",
        "def download_file_from_google_drive(id, destination):\n",
        "    def get_confirm_token(response):\n",
        "        for key, value in response.cookies.items():\n",
        "            if key.startswith('download_warning'):\n",
        "                return value\n",
        "\n",
        "        return None\n",
        "\n",
        "    def save_response_content(response, destination):\n",
        "        CHUNK_SIZE = 32768\n",
        "\n",
        "        with open(destination, \"wb\") as f:\n",
        "            for chunk in response.iter_content(CHUNK_SIZE):\n",
        "                if chunk: # filter out keep-alive new chunks\n",
        "                    f.write(chunk)\n",
        "\n",
        "    URL = \"https://docs.google.com/uc?export=download\"\n",
        "\n",
        "    session = requests.Session()\n",
        "\n",
        "    response = session.get(URL, params = { 'id' : id }, stream = True)\n",
        "    token = get_confirm_token(response)\n",
        "\n",
        "    if token:\n",
        "        params = { 'id' : id, 'confirm' : token }\n",
        "        response = session.get(URL, params = params, stream = True)\n",
        "\n",
        "    save_response_content(response, destination)    \n",
        "\n",
        "#Had to move the pretrained models to Google Drive as GitHub LFS has only 1GB limit bandwith for free accounts\n",
        "\n",
        "#download pretrained models for SSD Tensorflow\n",
        "file_id = \"15KS5tCSVRBiX_CB5y50LTTaq9jGhrmpN\"\n",
        "destination = \"checkpoints/ssd_300_vgg.ckpt.data-00000-of-00001\"\n",
        "\n",
        "download_file_from_google_drive(file_id, destination)\n",
        "file_id = \"1gYHVOgPo7ib3TO0ne3KxRn8nC0rdgXUZ\"\n",
        "destination = \"checkpoints/ssd_300_vgg.ckpt.index\"\n",
        "\n",
        "download_file_from_google_drive(file_id, destination)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZLZP55g-z8Zs",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "!ls -ls checkpoints\n",
        "\n",
        "from nets import ssd_vgg_300, ssd_common, np_methods"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "iNUIDHLeqehC",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "from preprocessing import ssd_vgg_preprocessing\n",
        "import visualization"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "52biFDKfqehI",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# TensorFlow session: grow memory when needed. TF, DO NOT USE ALL MY GPU MEMORY!!!\n",
        "gpu_options = tf.GPUOptions(allow_growth=True)\n",
        "config = tf.ConfigProto(log_device_placement=False, gpu_options=gpu_options)\n",
        "isess = tf.InteractiveSession(config=config)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cQhOmDAnqehN",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## SSD 300 Model\n",
        "\n",
        "The SSD 300 network takes 300x300 image inputs. In order to feed any image, the latter is resize to this input shape (i.e.`Resize.WARP_RESIZE`). Note that even though it may change the ratio width / height, the SSD model performs well on resized images (and it is the default behaviour in the original Caffe implementation).\n",
        "\n",
        "SSD anchors correspond to the default bounding boxes encoded in the network. The SSD net output provides offset on the coordinates and dimensions of these anchors."
      ]
    },
    {
      "metadata": {
        "id": "EeiwCpxbqehO",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# Input placeholder.\n",
        "net_shape = (300, 300)\n",
        "data_format = 'NCHW'\n",
        "img_input = tf.placeholder(tf.uint8, shape=(None, None, 3))\n",
        "# Evaluation pre-processing: resize to SSD net shape.\n",
        "image_pre, labels_pre, bboxes_pre, bbox_img = ssd_vgg_preprocessing.preprocess_for_eval(\n",
        "    img_input, None, None, net_shape, data_format, resize=ssd_vgg_preprocessing.Resize.WARP_RESIZE)\n",
        "image_4d = tf.expand_dims(image_pre, 0)\n",
        "\n",
        "# Define the SSD model.\n",
        "reuse = True if 'ssd_net' in locals() else None\n",
        "ssd_net = ssd_vgg_300.SSDNet()\n",
        "with slim.arg_scope(ssd_net.arg_scope(data_format=data_format)):\n",
        "    predictions, localisations, _, _ = ssd_net.net(image_4d, is_training=False, reuse=reuse)\n",
        "\n",
        "# Restore SSD model.\n",
        "ckpt_filename = './checkpoints/ssd_300_vgg.ckpt'\n",
        "# ckpt_filename = './checkpoints/VGG_VOC0712_SSD_300x300_ft_iter_120000.ckpt'\n",
        "isess.run(tf.global_variables_initializer())\n",
        "saver = tf.train.Saver()\n",
        "saver.restore(isess, ckpt_filename)\n",
        "\n",
        "# SSD default anchor boxes.\n",
        "ssd_anchors = ssd_net.anchors(net_shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "x7x9kKOtqehX",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Post-processing pipeline\n",
        "\n",
        "The SSD outputs need to be post-processed to provide proper detections. Namely, we follow these common steps:\n",
        "\n",
        "* Select boxes above a classification threshold;\n",
        "* Clip boxes to the image shape;\n",
        "* Apply the Non-Maximum-Selection algorithm: fuse together boxes whose Jaccard score > threshold;\n",
        "* If necessary, resize bounding boxes to original image shape."
      ]
    },
    {
      "metadata": {
        "id": "YNOsY-m5qeha",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# Main image processing routine.\n",
        "def process_image(img, select_threshold=0.5, nms_threshold=.45, net_shape=(300, 300)):\n",
        "    # Run SSD network.\n",
        "    rimg, rpredictions, rlocalisations, rbbox_img = isess.run([image_4d, predictions, localisations, bbox_img],\n",
        "                                                              feed_dict={img_input: img})\n",
        "    \n",
        "    # Get classes and bboxes from the net outputs.\n",
        "    rclasses, rscores, rbboxes = np_methods.ssd_bboxes_select(\n",
        "            rpredictions, rlocalisations, ssd_anchors,\n",
        "            select_threshold=select_threshold, img_shape=net_shape, num_classes=21, decode=True)\n",
        "    \n",
        "    rbboxes = np_methods.bboxes_clip(rbbox_img, rbboxes)\n",
        "    rclasses, rscores, rbboxes = np_methods.bboxes_sort(rclasses, rscores, rbboxes, top_k=400)\n",
        "    rclasses, rscores, rbboxes = np_methods.bboxes_nms(rclasses, rscores, rbboxes, nms_threshold=nms_threshold)\n",
        "    # Resize bboxes to original image shape. Note: useless for Resize.WARP!\n",
        "    rbboxes = np_methods.bboxes_resize(rbbox_img, rbboxes)\n",
        "    return rclasses, rscores, rbboxes"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XuljOiesQl-O",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "!rm video-test.mp4"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fmporqDF7Kda",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "import time\n",
        "\n",
        "start_time = time.time()\n",
        "\n",
        "#A smooth drive in The Crew on PS4 - OSSDC Simulator ACC Train 30fps\n",
        "url = 'https://www.youtube.com/watch?v=uuQlMCMT71I'\n",
        "\n",
        "#Bad weather sample, uncomment this line to see how the detection works on bad weather\n",
        "#url = \"https://www.youtube.com/watch?v=q3q26xrigX4\"\n",
        "\n",
        "#url= ... #put your Youtube video URL here and uncomment the line, to test on a different video\n",
        "\n",
        "def getVideoURL(url):\n",
        "    videoUrl = url\n",
        "    video = pafy.new(url)\n",
        "    streams = video.streams\n",
        "    videoUrlList={}\n",
        "    for s in streams:\n",
        "        videoUrlList[s.resolution] = s.url\n",
        "        #print(s.resolution, s.extension, s.get_filesize(), s.url)\n",
        "\n",
        "    if videoUrlList.get(\"1280x720\",None) is not None:\n",
        "        videoUrl = videoUrlList.get(\"1280x720\",None)\n",
        "        print(\"1280x720\")\n",
        "\n",
        "    if videoUrlList.get(\"1920x1080\",None) is not None:\n",
        "        videoUrl = videoUrlList.get(\"1920x1080\",None)\n",
        "        print(\"1920x1080\")\n",
        "    return videoUrl\n",
        "\n",
        "origVideoUrl = url\n",
        "\n",
        "if \"youtube.\" in url: \n",
        "    videoUrl = getVideoURL(url)\n",
        "    !wget -q -O video-test.mp4 '$videoUrl' #streaming directly doesn't work in Colaboratory yet, need to save the file on disk first\n",
        "else:\n",
        "    videoUrl = url\n",
        "\n",
        "print(\"videoUrl =\",videoUrl)\n",
        "videoUrl='video-test.mp4'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZTl8WQS3qehf",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "def draw_str(dst, target, s):\n",
        "    x, y = target\n",
        "    cv2.putText(dst, s, (x+1, y+1), cv2.FONT_HERSHEY_PLAIN, 1.0, (0, 0, 0), thickness = 2, lineType=cv2.LINE_AA)\n",
        "    cv2.putText(dst, s, (x, y), cv2.FONT_HERSHEY_PLAIN, 1.0, (255, 255, 255), lineType=cv2.LINE_AA)\n",
        "\n",
        "videoUrl='video-test.mp4'\n",
        "#process the downloaded video, see the box around the detected objects - cars, people etc\n",
        "\n",
        "webcam=False\n",
        "#webcam=True\n",
        "\n",
        "if webcam:\n",
        "    cap = WebcamVideoStream(videoUrl).start()\n",
        "else:\n",
        "    cap = cv2.VideoCapture(videoUrl)\n",
        "\n",
        "count=50\n",
        "skip=0\n",
        "SKIP_EVERY=150 #pick a frame every 5 seconds\n",
        "\n",
        "count=1000 #look only at first 1000 frames\n",
        "skip=0\n",
        "SKIP_EVERY=0\n",
        "\n",
        "every=SKIP_EVERY\n",
        "initial_time = getCurrentClock()\n",
        "flag=True\n",
        "\n",
        "frameCnt=0\n",
        "prevFrameCnt=0\n",
        "prevTime=getCurrentClock()\n",
        "\n",
        "showImage=False\n",
        "showImage=True\n",
        "\n",
        "\n",
        "streamImage=False\n",
        "#streamImage=True #uncomment this to stream the output to your PC, with ffplay as receiver\n",
        "\n",
        "processImage=False\n",
        "processImage=True\n",
        "\n",
        "zoomImage=0\n",
        "#zoomImage=4\n",
        "\n",
        "rclasses = []\n",
        "rscores = []\n",
        "rbboxes = []\n",
        "\n",
        "record = False\n",
        "#record = True #uncomment line to get a video with detections marked in place\n",
        "\n",
        "procWidth = 1280 #640   # processing width (x resolution) of frame\n",
        "procHeight = 720   # processing width (x resolution) of frame\n",
        "\n",
        "#procWidth = 640 #640   # processing width (x resolution) of frame\n",
        "#procHeight = 360   # processing width (x resolution) of frame\n",
        "\n",
        "#procWidth = 320 #640   # processing width (x resolution) of frame\n",
        "#procHeight = 180   # processing width (x resolution) of frame\n",
        "\n",
        "import socket\n",
        "\n",
        "if streamImage:\n",
        "  #on server run this command:\n",
        "  #ffplay -f mjpeg tcp://0.0.0.0:45654?listen\n",
        "  clientsocket=socket.socket(socket.AF_INET,socket.SOCK_STREAM)\n",
        "  clientsocket.connect(('put_your_target_ip_here',45654)) #the target ip is where the ffplay is listening\n",
        "\n",
        "out = None\n",
        "if record:\n",
        "    fourcc = cv2.VideoWriter_fourcc(*'MPEG')\n",
        "    timestr = time.strftime(\"%Y%m%d-%H%M%S\")\n",
        "    out = cv2.VideoWriter('output-'+timestr+'.mp4',fourcc, 30.0, (int(procWidth),int(procHeight)))\n",
        "frame = cap.read()\n",
        "\n",
        "fpsValue=0\n",
        "tracks=[]\n",
        "\n",
        "print(\"Start procesing!\",frame)\n",
        "try:\n",
        "    while True:\n",
        "        if webcam or cap.grab():\n",
        "            if webcam:\n",
        "                frame = cap.read()\n",
        "            else:\n",
        "                flag, frame = cap.retrieve()    \n",
        "            if not flag:\n",
        "                continue\n",
        "            else:\n",
        "                #print(\"frameCnt:\",frameCnt)\n",
        "                frame = cv2.resize(frame,(int(procWidth),int(procHeight)))\n",
        "                frameCnt=frameCnt+1\n",
        "                nowMicro = getCurrentClock()\n",
        "                delta = (nowMicro-prevTime).total_seconds()\n",
        "                #print(\"%f \" % (delta))\n",
        "                if delta>=1.0:\n",
        "                    fpsValue = ((frameCnt-prevFrameCnt)/delta) \n",
        "                    #print(\"FPS = %0.4f\" % ((frameCnt-prevFrameCnt)/delta))\n",
        "                    prevTime = nowMicro\n",
        "                    prevFrameCnt=frameCnt\n",
        "\n",
        "                if skip>0:\n",
        "                    skip=skip-1\n",
        "                    continue\n",
        "\n",
        "                if every>0:\n",
        "                    every=every-1\n",
        "                    continue\n",
        "                every=SKIP_EVERY\n",
        "\n",
        "                count=count-1\n",
        "                if count==0:\n",
        "                    break\n",
        "\n",
        "                img = frame\n",
        "                if processImage:    \n",
        "                    if zoomImage>0:\n",
        "                        #crop center of image, crop width is output_side_length\n",
        "                        output_side_length = int(procWidth/zoomImage)\n",
        "                        height, width, depth = frame.shape\n",
        "                        #print (height, width, depth)\n",
        "                        height_offset = int((height - output_side_length) / 2)\n",
        "                        width_offset = int((width - output_side_length) / 2)\n",
        "                        #print (height, width, depth, height_offset,width_offset,output_side_length)\n",
        "                        img = frame[height_offset:height_offset + output_side_length,width_offset:width_offset + output_side_length]\n",
        "\n",
        "                    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
        "                    start_time = getCurrentClock()\n",
        "                    rclasses, rscores, rbboxes =  process_image(img)\n",
        "                    if len(rclasses)>0:\n",
        "                        nowMicro = getCurrentClock()\n",
        "                        print(\"# %s - %s - %0.4f seconds ---\" % (frameCnt,rclasses.astype('|S3'), (nowMicro - start_time).total_seconds()))\n",
        "                        draw_str(img, (20, 20), \"FPS = %3.2f, %0.4f sec, Track points = %5d, Frame = %6d\" % (fpsValue,(nowMicro - start_time).total_seconds(),len(tracks), frameCnt))\n",
        "                        start_time = nowMicro\n",
        "                    \n",
        "                    #img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n",
        "                    visualization.bboxes_draw_on_img(img, rclasses, rscores, rbboxes, visualization.colors_plasma)\n",
        "                if showImage:\n",
        "                    imshow(img)\n",
        "                    show()\n",
        "                    if count % 3: #increase or decrease this to clear after more or less frames\n",
        "                      # Display the frame until new frame is available\n",
        "                      clear_output(wait=True)\n",
        "                if streamImage:\n",
        "                    if processImage:\n",
        "                        img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n",
        "                    data = cv2.imencode('.jpg', img)[1].tobytes()\n",
        "                    clientsocket.send(data)\n",
        "                if record:\n",
        "                    if processImage:\n",
        "                        img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n",
        "                    newimage = cv2.resize(img,(procWidth,procHeight))\n",
        "                    out.write(newimage)\n",
        "except KeyboardInterrupt:\n",
        "    # Release the Video Device\n",
        "    vid.release()\n",
        "    # Message to be displayed after releasing the device\n",
        "    print (\"Released Video Resource\")\n",
        "    \n",
        "nowMicro = getCurrentClock()\n",
        "print(\"# %s -- %0.4f seconds - FPS: %0.4f ---\" % (frameCnt, (nowMicro - initial_time).total_seconds(), frameCnt/(nowMicro - initial_time).total_seconds()))\n",
        "cap.release()\n",
        "if record:\n",
        "    out.release()\n",
        "if streamImage:\n",
        "    clientsocket.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "K5jv-3RXqeh5",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "!ls -la\n",
        "!pwd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "OpEETR2DaKzb",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}