Merge branch 'image_manip_refactor' into develop

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "adc0375d7ec059f253627e076edc4aab7f7a1a6b")
+set(DEPTHAI_DEVICE_SIDE_COMMIT "e1f13293c1e8215d65550d8fb5f30b80db5011c1")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

examples/CMakeLists.txt

@@ -239,6 +239,7 @@ dai_add_example(image_manip ImageManip/image_manip_example.cpp ON)
 dai_add_example(image_manip_rotate ImageManip/image_manip_rotate.cpp ON)
 dai_add_example(image_manip_tiling ImageManip/image_manip_tiling.cpp ON)
 dai_add_example(rgb_rotate_warp ImageManip/rgb_rotate_warp.cpp ON)
+dai_add_example(image_manip_warp_mesh ImageManip/image_manip_warp_mesh.cpp ON)
 
 # IMU
 dai_add_example(imu_gyroscope_accelerometer IMU/imu_gyroscope_accelerometer.cpp ON)

examples/ImageManip/image_manip_tiling.cpp

@@ -18,11 +18,15 @@ int main() {
     // preview frame into 2 500x500 frames
     auto manip1 = pipeline.create<dai::node::ImageManip>();
     manip1->initialConfig.setCropRect(0, 0, 0.5, 1);
+    // Flip functionality
+    manip1->initialConfig.setHorizontalFlip(true);
     manip1->setMaxOutputFrameSize(maxFrameSize);
     camRgb->preview.link(manip1->inputImage);
 
     auto manip2 = pipeline.create<dai::node::ImageManip>();
     manip2->initialConfig.setCropRect(0.5, 0, 1, 1);
+    // Flip functionality
+    manip1->initialConfig.setVerticalFlip(true);
     manip2->setMaxOutputFrameSize(maxFrameSize);
     camRgb->preview.link(manip2->inputImage);
 

examples/ImageManip/image_manip_warp_mesh.cpp

@@ -0,0 +1,68 @@
+#include <iostream>
+
+// Inludes common necessary includes for development using depthai library
+#include "depthai/depthai.hpp"
+
+int main() {
+    using namespace std;
+
+    // Create pipeline
+    dai::Pipeline pipeline;
+
+    auto camRgb = pipeline.create<dai::node::ColorCamera>();
+    camRgb->setPreviewSize(500, 500);
+    camRgb->setInterleaved(false);
+    auto maxFrameSize = camRgb->getPreviewWidth() * camRgb->getPreviewHeight() * 3;
+
+    // Warp preview frame 1
+    auto manip1 = pipeline.create<dai::node::ImageManip>();
+    // Create a custom warp mesh
+    dai::Point2f tl(20, 20);
+    dai::Point2f tr(460, 20);
+    dai::Point2f ml(100, 250);
+    dai::Point2f mr(400, 250);
+    dai::Point2f bl(20, 460);
+    dai::Point2f br(460, 460);
+    manip1->setWarpMesh({tl, tr, ml, mr, bl, br}, 2, 3);
+    manip1->setMaxOutputFrameSize(maxFrameSize);
+
+    camRgb->preview.link(manip1->inputImage);
+    auto xout1 = pipeline.create<dai::node::XLinkOut>();
+    xout1->setStreamName("out1");
+    manip1->out.link(xout1->input);
+
+    // Warp preview frame 2
+    auto manip2 = pipeline.create<dai::node::ImageManip>();
+    // Create a custom warp mesh
+    // clang-format off
+    std::vector<dai::Point2f> mesh2 = {
+        {20, 20}, {250, 100}, {460, 20},
+        {100,250}, {250, 250}, {400, 250},
+        {20, 480}, {250,400}, {460,480}
+    };
+    // clang-format on
+    manip2->setWarpMesh(mesh2, 3, 3);
+    manip2->setMaxOutputFrameSize(maxFrameSize);
+
+    camRgb->preview.link(manip2->inputImage);
+    auto xout2 = pipeline.create<dai::node::XLinkOut>();
+    xout2->setStreamName("out2");
+    manip2->out.link(xout2->input);
+
+    dai::Device device(pipeline);
+    auto q1 = device.getOutputQueue("out1", 8, false);
+    auto q2 = device.getOutputQueue("out2", 8, false);
+    while(true) {
+        auto in1 = q1->get<dai::ImgFrame>();
+        if(in1) {
+            cv::imshow("Warped preview 1", in1->getCvFrame());
+        }
+        auto in2 = q2->get<dai::ImgFrame>();
+        if(in2) {
+            cv::imshow("Warped preview 2", in2->getCvFrame());
+        }
+        int key = cv::waitKey(1);
+        if(key == 'q' || key == 'Q') return 0;
+    }
+    return 0;
+}

include/depthai/pipeline/datatype/ImageManipConfig.hpp

@@ -156,6 +156,12 @@ class ImageManipConfig : public Buffer {
      */
     ImageManipConfig& setHorizontalFlip(bool flip);
 
+    /**
+     * Specify vertical flip
+     * @param flip True to enable vertical flip, false otherwise
+     */
+    void setVerticalFlip(bool flip);
+
     /**
      * Instruct ImageManip to not remove current image from its queue and use the same for next message.
      * @param reuse True to enable reuse, false otherwise

include/depthai/pipeline/node/ImageManip.hpp

@@ -22,6 +22,8 @@ class ImageManip : public NodeCRTP<Node, ImageManip, ImageManipProperties> {
    private:
     std::shared_ptr<RawImageManipConfig> rawConfig;
 
+    void setWarpMesh(const float* meshData, int numMeshPoints, int width, int height);
+
    public:
     ImageManip(const std::shared_ptr<PipelineImpl>& par, int64_t nodeId);
     ImageManip(const std::shared_ptr<PipelineImpl>& par, int64_t nodeId, std::unique_ptr<Properties> props);
@@ -53,7 +55,7 @@ class ImageManip : public NodeCRTP<Node, ImageManip, ImageManipProperties> {
     [[deprecated("Use 'initialConfig.setCenterCrop()' instead")]] void setCenterCrop(float ratio, float whRatio = 1.0f);
     [[deprecated("Use 'initialConfig.setResize()' instead")]] void setResize(int w, int h);
     [[deprecated("Use 'initialConfig.setResizeThumbnail()' instead")]] void setResizeThumbnail(int w, int h, int bgRed = 0, int bgGreen = 0, int bgBlue = 0);
-    [[deprecated("Use 'initialConfig.setFrameType()' instead")]] void setFrameType(dai::RawImgFrame::Type name);
+    [[deprecated("Use 'initialConfig.setFrameType()' instead")]] void setFrameType(ImgFrame::Type name);
     [[deprecated("Use 'initialConfig.setHorizontalFlip()' instead")]] void setHorizontalFlip(bool flip);
     void setKeepAspectRatio(bool keep);
 
@@ -81,6 +83,15 @@ class ImageManip : public NodeCRTP<Node, ImageManip, ImageManipProperties> {
      * @param maxFrameSize Maximum frame size in bytes
      */
     void setMaxOutputFrameSize(int maxFrameSize);
+
+    /**
+     * Set a custom warp mesh
+     * @param meshData 2D plane of mesh points, starting from top left to bottom right
+     * @param width Width of mesh
+     * @param height Height of mesh
+     */
+    void setWarpMesh(const std::vector<Point2f>& meshData, int width, int height);
+    void setWarpMesh(const std::vector<std::pair<float, float>>& meshData, int width, int height);
 };
 
 }  // namespace node

src/pipeline/datatype/ImageManipConfig.cpp

@@ -169,6 +169,14 @@ ImageManipConfig& ImageManipConfig::setHorizontalFlip(bool flip) {
     return *this;
 }
 
+void ImageManipConfig::setVerticalFlip(bool flip) {
+    // Enable format stage
+    cfg.enableFormat = true;
+
+    // Set pixel format
+    cfg.formatConfig.flipVertical = flip;
+}
+
 ImageManipConfig& ImageManipConfig::setReusePreviousImage(bool reuse) {
     cfg.reusePreviousImage = reuse;
     return *this;

src/pipeline/node/ImageManip.cpp

@@ -69,5 +69,52 @@ void ImageManip::setMaxOutputFrameSize(int maxFrameSize) {
     properties.outputFrameSize = maxFrameSize;
 }
 
+void ImageManip::setWarpMesh(const float* meshData, int numMeshPoints, int width, int height) {
+    if(numMeshPoints < width * height) {
+        throw std::invalid_argument("Not enough points provided for specified width and height");
+    }
+
+    // TODO(themarpe) - optimize
+    Asset asset("mesh");
+    asset.alignment = 64;
+
+    // Align stride to 16B
+    constexpr auto ALIGNMENT = 16;
+    size_t meshStride = ((size_t)((sizeof(Point2f) * width)) + (ALIGNMENT - 1)) & ~(ALIGNMENT - 1);
+    // Specify final mesh size
+    size_t meshSize = meshStride * height;
+
+    // Create mesh data
+    asset.data = std::vector<uint8_t>(meshSize);
+
+    // Fill out mesh points with stride
+    for(int i = 0; i < height; i++) {
+        for(int j = 0; j < width; j++) {
+            // get location in meshData
+            size_t inputMeshIndex = (i * width + j) * 2;  // 2 float values per point
+
+            // get output offset
+            size_t outputMeshOffset = (meshStride * i) + (j * sizeof(Point2f));
+            auto& point = reinterpret_cast<Point2f&>(asset.data.data()[outputMeshOffset]);
+
+            // Asign reversed mesh coordinates (HW specified)
+            point.x = meshData[inputMeshIndex + 1];
+            point.y = meshData[inputMeshIndex + 0];
+        }
+    }
+
+    properties.meshUri = assetManager.set(asset)->getRelativeUri();
+    properties.meshWidth = width;
+    properties.meshHeight = height;
+}
+
+void ImageManip::setWarpMesh(const std::vector<Point2f>& meshData, int width, int height) {
+    setWarpMesh(reinterpret_cast<const float*>(meshData.data()), static_cast<int>(meshData.size()), width, height);
+}
+
+void ImageManip::setWarpMesh(const std::vector<std::pair<float, float>>& meshData, int width, int height) {
+    setWarpMesh(reinterpret_cast<const float*>(meshData.data()), static_cast<int>(meshData.size()), width, height);
+}
+
 }  // namespace node
 }  // namespace dai