Better OpenCV integration

CMakeLists.txt

@@ -598,6 +598,7 @@ option(BUILD_UNIT_TESTS "Build realsense unit tests." ON)
 option(BUILD_EXAMPLES "Build realsense examples and tools." ON)
 option(ENFORCE_METADATA "Require WinSDK with Metadata support during compilation. Windows OS Only" OFF)
 option(BUILD_PYTHON_BINDINGS "Build Python bindings" OFF)
+option(BUILD_CV_EXAMPLES "Build OpenCV examples" OFF)
 option(BUILD_NODEJS_BINDINGS "Build Node.js bindings" OFF)
 
 # This parameter is meant for disabling graphical examples when building for
@@ -629,6 +630,10 @@ if (BUILD_NODEJS_BINDINGS)
   add_subdirectory(wrappers/nodejs)
 endif()
 
+if (BUILD_CV_EXAMPLES)
+  add_subdirectory(wrappers/opencv)
+endif()
+
 # Check for unreferenced files
 FILE(GLOB_RECURSE AllSources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
   "src/*.c" "src/*.cpp" "src/*.cc" "src/*.c++"

readme.md

@@ -37,7 +37,7 @@ Information about the Intel® RealSense™ technology at [realsense.intel.com](h
    These simple examples demonstrate how to easily use the SDK to include code snippets that access the camera into your applications.
 
 **[Wrappers](https://github.com/IntelRealSense/librealsense/tree/development/wrappers)**
-  We provide a C, C++, [Python](./wrappers/python), [Node.js](./wrappers/nodejs) API, [ROS](https://github.com/intel-ros/realsense/releases) and [LabVIEW](./wrappers/labview). More to come, including C# and Matlab.
+  We provide a C, C++, [Python](./wrappers/python), [Node.js](./wrappers/nodejs) API, [ROS](https://github.com/intel-ros/realsense/releases), [LabVIEW](./wrappers/labview) and [OpenCV](./wrappers/opencv) integration. More to come, including C#, Unity and Matlab.
 
 
 ## Quick Start

wrappers/opencv/CMakeLists.txt

@@ -0,0 +1,27 @@
+#  minimum required cmake version: 3.1.0
+cmake_minimum_required(VERSION 3.1.0)
+
+project(RealsenseCVExamples)
+
+# Save the command line compile commands in the build output
+set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
+# View the makefile commands during build
+#set(CMAKE_VERBOSE_MAKEFILE on)
+
+include(CheckCXXCompilerFlag)
+CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
+CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X)
+if(COMPILER_SUPPORTS_CXX11)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+elseif(COMPILER_SUPPORTS_CXX0X)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
+endif()
+
+find_package(OpenCV REQUIRED)
+
+set(DEPENDENCIES realsense2 ${OpenCV_LIBS})
+
+add_subdirectory(imshow)
+add_subdirectory(grabcuts)
+add_subdirectory(latency-tool)
+add_subdirectory(dnn)

wrappers/opencv/cv-helpers.hpp

@@ -0,0 +1,57 @@
+// License: Apache 2.0. See LICENSE file in root directory.
+// Copyright(c) 2017 Intel Corporation. All Rights Reserved.
+
+#pragma once
+
+#include <librealsense2/rs.hpp> // Include RealSense Cross Platform API
+#include <opencv2/opencv.hpp>   // Include OpenCV API
+#include <exception>
+
+// Convert rs2::frame to cv::Mat
+cv::Mat frame_to_mat(const rs2::frame& f)
+{
+    using namespace cv;
+    using namespace rs2;
+
+    auto vf = f.as<video_frame>();
+    const int w = vf.get_width();
+    const int h = vf.get_height();
+
+    if (f.get_profile().format() == RS2_FORMAT_BGR8)
+    {
+        return Mat(Size(w, h), CV_8UC3, (void*)f.get_data(), Mat::AUTO_STEP);
+    }
+    else if (f.get_profile().format() == RS2_FORMAT_RGB8)
+    {
+        auto r = Mat(Size(w, h), CV_8UC3, (void*)f.get_data(), Mat::AUTO_STEP);
+        cv::cvtColor(r, r, CV_BGR2RGB);
+        return r;
+    }
+    else if (f.get_profile().format() == RS2_FORMAT_Z16)
+    {
+        return Mat(Size(w, h), CV_16UC1, (void*)f.get_data(), Mat::AUTO_STEP);
+    }
+    else if (f.get_profile().format() == RS2_FORMAT_Y8)
+    {
+        return Mat(Size(w, h), CV_8UC1, (void*)f.get_data(), Mat::AUTO_STEP);;
+    }
+
+    throw std::runtime_error("Frame format is not supported yet!");
+}
+
+// Converts depth frame to a matrix of doubles with distances in meters
+cv::Mat depth_frame_to_meters(const rs2::pipeline& pipe, const rs2::depth_frame& f)
+{
+    using namespace cv;
+    using namespace rs2;
+
+    Mat dm = frame_to_mat(f);
+    dm.convertTo(dm, CV_64F);
+    auto depth_scale = pipe.get_active_profile()
+        .get_device()
+        .first<depth_sensor>()
+        .get_depth_scale();
+    dm = dm * depth_scale;
+    return dm;
+}
+

wrappers/opencv/dnn/CMakeLists.txt

@@ -0,0 +1,50 @@
+#  minimum required cmake version: 3.1.0
+cmake_minimum_required(VERSION 3.1.0)
+
+project(RealSenseDNNExample)
+
+# Save the command line compile commands in the build output
+set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
+
+include(CheckCXXCompilerFlag)
+CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
+CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X)
+if(COMPILER_SUPPORTS_CXX11)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+elseif(COMPILER_SUPPORTS_CXX0X)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
+endif()
+
+add_executable(rs-dnn rs-dnn.cpp ../cv-helpers.hpp)
+target_link_libraries(rs-dnn ${DEPENDENCIES})
+set_target_properties (rs-dnn PROPERTIES
+	FOLDER "Examples/OpenCV"
+)
+
+set(PROTOTEXT_FILE "MobileNetSSD_deploy.prototxt")
+set(PROTOTEXT_URL "https://raw.githubusercontent.com/chuanqi305/MobileNet-SSD/master/MobileNetSSD_deploy.prototxt")
+set(PROTOTEXT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${PROTOTEXT_FILE}")
+
+set(MODEL_FILE "MobileNetSSD_deploy.caffemodel")
+set(MODEL_URL "https://drive.google.com/uc?export=download&id=0B3gersZ2cHIxRm5PMWRoTkdHdHc")
+set(MODEL_PATH "${CMAKE_CURRENT_BINARY_DIR}/${MODEL_FILE}")
+
+if(NOT EXISTS "${PROTOTEXT_PATH}")
+	message("Downloading ${PROTOTEXT_FILE} into ${CMAKE_CURRENT_BINARY_DIR}")
+    file(DOWNLOAD "${PROTOTEXT_URL}" "${PROTOTEXT_PATH}")
+endif()
+
+if(NOT EXISTS "${MODEL_PATH}")
+	message("Downloading ${MODEL_FILE} into ${CMAKE_CURRENT_BINARY_DIR}")
+    file(DOWNLOAD "${MODEL_URL}" "${MODEL_PATH}")
+endif()
+
+install(
+	TARGETS
+
+	rs-dnn
+
+	RUNTIME DESTINATION
+	${CMAKE_INSTALL_PREFIX}/bin
+)

wrappers/opencv/dnn/readme.md

@@ -0,0 +1,17 @@
+# rs-dnn Sample
+
+## Overview
+This example shows how to use Intel RealSense cameras with existing [Deep Neural Network](https://en.wikipedia.org/wiki/Deep_learning) algorithms. The demo is derived from [MobileNet Single-Shot Detector example](https://github.com/opencv/opencv/blob/master/samples/dnn/ssd_mobilenet_object_detection.cpp) provided with `opencv`. We modify it to work with Intel RealSense cameras and take advantage of depth data (in a very basic way). 
+
+
+The demo will load existing [Caffe model](https://github.com/chuanqi305/MobileNet-SSD) (see another tutorial [here](https://docs.opencv.org/3.3.0/d5/de7/tutorial_dnn_googlenet.html)) and use it to classify objects within the RGB image. Once object is detected, the demo will calculate approximate distance to the object using the depth data:
+
+<p align="center"><img src="res/1.PNG" /></p>
+
+## Implementation Details
+
+Unlike the other samples, this demo requires access to the exact depth values. We generate a matrix of floating point values (in meters) using the following helper function:
+```cpp
+auto depth_mat = depth_frame_to_meters(pipe, depth_frame);
+```
+

wrappers/opencv/dnn/rs-dnn.cpp

@@ -0,0 +1,148 @@
+// This example is derived from the ssd_mobilenet_object_detection opencv demo
+// and adapted to be used with Intel RealSense Cameras
+// Please see https://github.com/opencv/opencv/blob/master/LICENSE
+
+#include <opencv2/dnn.hpp>
+#include <librealsense2/rs.hpp>
+#include "../cv-helpers.hpp"
+
+const size_t inWidth      = 300;
+const size_t inHeight     = 300;
+const float WHRatio       = inWidth / (float)inHeight;
+const float inScaleFactor = 0.007843f;
+const float meanVal       = 127.5;
+const char* classNames[]  = {"background",
+                             "aeroplane", "bicycle", "bird", "boat",
+                             "bottle", "bus", "car", "cat", "chair",
+                             "cow", "diningtable", "dog", "horse",
+                             "motorbike", "person", "pottedplant",
+                             "sheep", "sofa", "train", "tvmonitor"};
+
+int main(int argc, char** argv) try
+{
+    using namespace rs2;
+    using namespace cv;
+    using namespace cv::dnn;
+
+    Net net = readNetFromCaffe("MobileNetSSD_deploy.prototxt", 
+                               "MobileNetSSD_deploy.caffemodel");
+
+    // Start streaming from Intel RealSense Camera
+    pipeline pipe;
+    auto config = pipe.start();
+    auto profile = config.get_stream(RS2_STREAM_COLOR)
+                         .as<video_stream_profile>();
+    rs2::align align_to(RS2_STREAM_COLOR);
+
+    Size cropSize;
+    if (profile.width() / (float)profile.height() > WHRatio)
+    {
+        cropSize = Size(static_cast<int>(profile.height() * WHRatio),
+                        profile.height());
+    }
+    else
+    {
+        cropSize = Size(profile.width(),
+                        static_cast<int>(profile.width() / WHRatio));
+    }
+
+    Rect crop(Point((profile.width() - cropSize.width) / 2,
+                    (profile.height() - cropSize.height) / 2),
+              cropSize);
+
+    const auto window_name = "Display Image";
+    namedWindow(window_name, WINDOW_AUTOSIZE);
+
+    while (cvGetWindowHandle(window_name))
+    {
+        // Wait for the next set of frames
+        auto data = pipe.wait_for_frames();
+        // Make sure the frames are spatially aligned
+        data = align_to.proccess(data); 
+
+        auto color_frame = data.get_color_frame();
+        auto depth_frame = data.get_depth_frame();
+
+        // If we only received new depth frame, 
+        // but the color did not update, continue
+        static int last_frame_number = 0;
+        if (color_frame.get_frame_number() == last_frame_number) continue;
+        last_frame_number = color_frame.get_frame_number();
+
+        // Convert RealSense frame to OpenCV matrix:
+        auto color_mat = frame_to_mat(color_frame);
+        auto depth_mat = depth_frame_to_meters(pipe, depth_frame);
+
+        Mat inputBlob = blobFromImage(color_mat, inScaleFactor,
+                                      Size(inWidth, inHeight), meanVal, false); //Convert Mat to batch of images
+        net.setInput(inputBlob, "data"); //set the network input
+        Mat detection = net.forward("detection_out"); //compute output
+
+        Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
+
+        // Crop both color and depth frames
+        color_mat = color_mat(crop);
+        depth_mat = depth_mat(crop);
+
+        float confidenceThreshold = 0.8f;
+        for(int i = 0; i < detectionMat.rows; i++)
+        {
+            float confidence = detectionMat.at<float>(i, 2);
+
+            if(confidence > confidenceThreshold)
+            {
+                size_t objectClass = (size_t)(detectionMat.at<float>(i, 1));
+
+                int xLeftBottom = static_cast<int>(detectionMat.at<float>(i, 3) * color_mat.cols);
+                int yLeftBottom = static_cast<int>(detectionMat.at<float>(i, 4) * color_mat.rows);
+                int xRightTop = static_cast<int>(detectionMat.at<float>(i, 5) * color_mat.cols);
+                int yRightTop = static_cast<int>(detectionMat.at<float>(i, 6) * color_mat.rows);
+
+                Rect object((int)xLeftBottom, (int)yLeftBottom,
+                            (int)(xRightTop - xLeftBottom),
+                            (int)(yRightTop - yLeftBottom));
+
+                object = object  & cv::Rect(0, 0, depth_mat.cols, depth_mat.rows);
+
+                // Calculate mean depth inside the detection region
+                // This is a very naive way to estimate objects depth
+                // but it is intended to demonstrate how one might 
+                // use depht data in general
+                Scalar m = mean(depth_mat(object));
+
+                std::ostringstream ss;
+                ss << classNames[objectClass] << " ";
+                ss << std::setprecision(2) << m[0] << " meters away";
+                String conf(ss.str());
+
+                rectangle(color_mat, object, Scalar(0, 255, 0));
+                int baseLine = 0;
+                Size labelSize = getTextSize(ss.str(), FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
+
+                auto center = (object.br() + object.tl())*0.5;
+                center.x = center.x - labelSize.width / 2;
+
+                rectangle(color_mat, Rect(Point(center.x, center.y - labelSize.height),
+                    Size(labelSize.width, labelSize.height + baseLine)),
+                    Scalar(255, 255, 255), CV_FILLED);
+                putText(color_mat, ss.str(), center,
+                        FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0,0,0));
+            }
+        }
+
+        imshow(window_name, color_mat);
+        if (waitKey(1) >= 0) break;
+    }
+
+    return EXIT_SUCCESS;
+}
+catch (const rs2::error & e)
+{
+    std::cerr << "RealSense error calling " << e.get_failed_function() << "(" << e.get_failed_args() << "):\n    " << e.what() << std::endl;
+    return EXIT_FAILURE;
+}
+catch (const std::exception& e)
+{
+    std::cerr << e.what() << std::endl;
+    return EXIT_FAILURE;
+}

wrappers/opencv/grabcuts/CMakeLists.txt

@@ -0,0 +1,32 @@
+#  minimum required cmake version: 3.1.0
+cmake_minimum_required(VERSION 3.1.0)
+
+project(RealSenseGrabCutsExample)
+
+# Save the command line compile commands in the build output
+set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
+
+include(CheckCXXCompilerFlag)
+CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
+CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X)
+if(COMPILER_SUPPORTS_CXX11)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+elseif(COMPILER_SUPPORTS_CXX0X)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
+endif()
+
+add_executable(rs-grabcuts rs-grabcuts.cpp ../cv-helpers.hpp)
+target_link_libraries(rs-grabcuts ${DEPENDENCIES})
+set_target_properties (rs-grabcuts PROPERTIES
+	FOLDER "Examples/OpenCV"
+)
+
+install(
+	TARGETS
+
+	rs-grabcuts
+
+	RUNTIME DESTINATION
+	${CMAKE_INSTALL_PREFIX}/bin
+)