Refactor aidemo

package/ai/ai.mk

@@ -41,17 +41,17 @@ define AI_COMPILE_MODEL
 	#tar -xf $(AI_DL_DIR)/kmodel.tar.xz -C  $(TARGET_DIR)/app/ai
 
 	#demo7
-	mkdir -p $(TARGET_DIR)/app/ai/kmodel/kmodel_compile/object_detect
-	cd $(@D) && /usr/bin/python3 object_detect_opencv/od_onnx.py --quant_type uint8 --model ai_kmodel_data/model_file/object_detect/yolov5sFocus_320x3.onnx
-	cp $(@D)/od.kmodel $(TARGET_DIR)/app/ai/kmodel/kmodel_compile/object_detect/od_uint8.kmodel
-	cd $(@D) && /usr/bin/python3 object_detect_opencv/od_onnx.py --quant_type bf16 --model ai_kmodel_data/model_file/object_detect/yolov5sFocus_320x3.onnx
-	cp $(@D)/od.kmodel $(TARGET_DIR)/app/ai/kmodel/kmodel_compile/object_detect/od_bf16.kmodel
+	mkdir -p $(TARGET_DIR)/app/ai/kmodel/kmodel_compile/object_detect_demo
+	cd $(@D) && /usr/bin/python3 object_detect_demo/od_onnx.py --quant_type uint8 --model ai_kmodel_data/model_file/object_detect/yolov5sFocus_320x3.onnx
+	cp $(@D)/od.kmodel $(TARGET_DIR)/app/ai/kmodel/kmodel_compile/object_detect_demo/od_uint8.kmodel
+	cd $(@D) && /usr/bin/python3 object_detect_demo/od_onnx.py --quant_type bf16 --model ai_kmodel_data/model_file/object_detect/yolov5sFocus_320x3.onnx
+	cp $(@D)/od.kmodel $(TARGET_DIR)/app/ai/kmodel/kmodel_compile/object_detect_demo/od_bf16.kmodel
 
 	#demo1
 	mkdir -p $(TARGET_DIR)/app/ai/kmodel/kmodel_compile/retinaface_mb_320
-	cd $(@D) && /usr/bin/python3 retinaface_mb_320_opencv/rf_onnx.py --quant_type uint8 --model ai_kmodel_data/model_file/retinaface/retinaface_mobile0.25_320.onnx
+	cd $(@D) && /usr/bin/python3 retinaface_mb_320/rf_onnx.py --quant_type uint8 --model ai_kmodel_data/model_file/retinaface/retinaface_mobile0.25_320.onnx
 	cp $(@D)/rf.kmodel $(TARGET_DIR)/app/ai/kmodel/kmodel_compile/retinaface_mb_320/rf_uint8.kmodel
-	cd $(@D) && /usr/bin/python3 retinaface_mb_320_opencv/rf_onnx.py --quant_type bf16 --model ai_kmodel_data/model_file/retinaface/retinaface_mobile0.25_320.onnx
+	cd $(@D) && /usr/bin/python3 retinaface_mb_320/rf_onnx.py --quant_type bf16 --model ai_kmodel_data/model_file/retinaface/retinaface_mobile0.25_320.onnx
 	cp $(@D)/rf.kmodel $(TARGET_DIR)/app/ai/kmodel/kmodel_compile/retinaface_mb_320/rf_bf16.kmodel
 endef
 

package/ai/code/CMakeLists.txt

@@ -21,9 +21,7 @@ add_subdirectory(person_detect)
 add_subdirectory(hand_image_classify)
 add_subdirectory(license_plate_recog)
 add_subdirectory(self_learning)
-add_subdirectory(object_detect_v4l2)
-add_subdirectory(retinaface_mb_320_v4l2)
-add_subdirectory(object_detect_opencv)
-add_subdirectory(retinaface_mb_320_opencv)
+add_subdirectory(object_detect_demo)
+add_subdirectory(retinaface_mb_320)
 
 

package/ai/code/object_detect_opencv/CMakeLists.txt → package/ai/code/object_detect_demo/CMakeLists.txt

@@ -1,6 +1,6 @@
-set(SRC main.cc cv2_utils.cc object_detect.cc ../common/k510_drm.c)
-set(bin object_detect_opencv)
-set(script ../shell/object_detect_opencv_bf16.sh ../shell/object_detect_opencv_uint8.sh)
+set(SRC main.cc cv2_utils.cc object_detect.cc ../common/k510_drm.c ../common/v4l2.c)
+set(bin object_detect_demo)
+set(script ../shell/object_detect_demo_bf16.sh ../shell/object_detect_demo_uint8.sh)
 set(input ../imx219_0.conf ../imx219_1.conf ../video_192x320.conf)
 
 

package/ai/code/object_detect_opencv/main.cc → package/ai/code/object_detect_demo/main.cc

@@ -55,6 +55,7 @@
 #include "k510_drm.h"
 #include "media_ctl.h"
 #include <linux/videodev2.h>
+#include "v4l2.h"
 
 #define PROFILING 0
 
@@ -83,6 +84,28 @@ void fun_sig(int sig)
     }
 }
 
+static int video_stop(struct v4l2_device *vdev)
+{
+	int ret;
+
+	ret = v4l2_stream_off(vdev);
+	if (ret < 0) {
+		printf("error: failed to stop video stream: %s (%d)\n",
+			strerror(-ret), ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void video_cleanup(struct v4l2_device *vdev)
+{
+	if (vdev) {
+		v4l2_free_buffers(vdev);
+		v4l2_close(vdev);
+	}
+}
+
 void ai_worker()
 {
     float obj_thresh = 0.2;
@@ -256,47 +279,152 @@ void ai_worker()
     }
 }
 
-void display_worker()
+static int process_ds0_image(struct v4l2_device *vdev,unsigned int width,unsigned int height)
 {
+	struct v4l2_video_buffer buffer;
+	int ret;
+    static struct v4l2_video_buffer old_buffer;
+    static int screen_init_flag = 0;
+
     mtx.lock();
-    cv::VideoCapture capture;
-	capture.open(3);
-    capture.set(cv::CAP_PROP_CONVERT_RGB, 0);
-    capture.set(cv::CAP_PROP_FRAME_WIDTH, dev_info[0].video_width[1]);
-    capture.set(cv::CAP_PROP_FRAME_HEIGHT, dev_info[0].video_height[1]);
-    capture.set(cv::CAP_PROP_FOURCC, V4L2_PIX_FMT_NV12);
+	ret = v4l2_dequeue_buffer(vdev, &buffer);
+	if (ret < 0) {
+		printf("error: unable to dequeue buffer: %s (%d)\n",
+			strerror(-ret), ret);
+        mtx.unlock();
+		return ret;
+	}
     mtx.unlock();
-    while(quit.load()) {
-        drm_bufs_index = !drm_bufs_index;
-        fbuf_yuv = &drm_dev.drm_bufs[drm_bufs_index];
-        cv::Mat org_img(DRM_INPUT_HEIGHT*3/2, (DRM_INPUT_WIDTH+15)/16*16, CV_8UC1, fbuf_yuv->map);
-        {
-            bool ret = false;
-#if PROFILING
-            ScopedTiming st("capture read");
-#endif
-            mtx.lock();
-            ret = capture.read(org_img);
+	if (buffer.error) {
+		printf("warning: error in dequeued buffer, skipping\n");
+		return 0;
+	}
+
+    fbuf_yuv = &drm_dev.drm_bufs[buffer.index];
+
+    if (drm_dev.req)
+        drm_wait_vsync();
+    fbuf_argb = &drm_dev.drm_bufs_argb[!drm_bufs_argb_index];//等待AI计算后的buffer
+    if (drm_dmabuf_set_plane(fbuf_yuv, fbuf_argb)) {
+        std::cerr << "Flush fail \n";
+        return 1;
+    }
+
+    if(screen_init_flag) {
+        fbuf_yuv = &drm_dev.drm_bufs[old_buffer.index];
+        old_buffer.mem = fbuf_yuv->map;
+        old_buffer.size = fbuf_yuv->size;
+        mtx.lock();
+        ret = v4l2_queue_buffer(vdev, &old_buffer);
+        if (ret < 0) {
+            printf("error: unable to requeue buffer: %s (%d)\n",
+                strerror(-ret), ret);
             mtx.unlock();
-            if(ret == false)
-            {
-                quit.store(false);
-                continue; // 
-            }
+            return ret;
         }
+        mtx.unlock();
+    }
+    else {
+        screen_init_flag = 1;
+    }
 
-        if (drm_dev.req)
-            drm_wait_vsync();
-        fbuf_argb = &drm_dev.drm_bufs_argb[!drm_bufs_argb_index];
-        if (drm_dmabuf_set_plane(fbuf_yuv, fbuf_argb)) {
-            std::cerr << "Flush fail \n";
-            goto exit;
-        }
+    old_buffer = buffer;
+
+	return 0;
+}
+
+void display_worker()
+{
+    int ret;
+    struct v4l2_device *vdev;
+    struct v4l2_pix_format format;
+    fd_set fds;
+    struct v4l2_video_buffer buffer;
+	unsigned int i;
+
+    mtx.lock();
+    vdev = v4l2_open(dev_info[0].video_name[1]);
+    if (vdev == NULL) {
+		printf("error: unable to open video capture device %s\n",
+			dev_info[0].video_name[1]);
+        mtx.unlock();
+		goto display_cleanup;
+	}
+
+	memset(&format, 0, sizeof format);
+	format.pixelformat = dev_info[0].video_out_format[1] ? V4L2_PIX_FMT_NV12 : V4L2_PIX_FMT_NV16;
+	format.width = dev_info[0].video_width[1];
+	format.height = dev_info[0].video_height[1];
+
+	ret = v4l2_set_format(vdev, &format);
+	if (ret < 0)
+	{
+		printf("%s:v4l2_set_format error\n",__func__);
+        mtx.unlock();
+		goto display_cleanup;
+	}
+
+	ret = v4l2_alloc_buffers(vdev, V4L2_MEMORY_USERPTR, DRM_BUFFERS_COUNT);
+	if (ret < 0)
+	{
+		printf("%s:v4l2_alloc_buffers error\n",__func__);
+        mtx.unlock();
+		goto display_cleanup;
+	}
+
+	FD_ZERO(&fds);
+	FD_SET(vdev->fd, &fds);
+
+	for (i = 0; i < vdev->nbufs; ++i) {
+		buffer.index = i;
+        fbuf_yuv = &drm_dev.drm_bufs[buffer.index];
+        buffer.mem = fbuf_yuv->map;
+        buffer.size = fbuf_yuv->size;
+
+		ret = v4l2_queue_buffer(vdev, &buffer);
+		if (ret < 0) {
+			printf("error: unable to queue buffer %u\n", i);
+            mtx.unlock();
+			goto display_cleanup;
+		}	
+	}
+
+	ret = v4l2_stream_on(vdev);
+	if (ret < 0) {
+		printf("%s error: failed to start video stream: %s (%d)\n", __func__,
+			strerror(-ret), ret);
+        mtx.unlock();
+		goto display_cleanup;
+	}
+    mtx.unlock();
+
+    while(quit.load()) {
+		struct timeval timeout;
+		fd_set rfds;
+
+		timeout.tv_sec = SELECT_TIMEOUT / 1000;
+		timeout.tv_usec = (SELECT_TIMEOUT % 1000) * 1000;
+		rfds = fds;
+
+		ret = select(vdev->fd + 1, &rfds, NULL, NULL, &timeout);
+		if (ret < 0) {
+			if (errno == EINTR)
+				continue;
+
+			printf("error: select failed with %d\n", errno);
+			goto display_cleanup;
+		}
+		if (ret == 0) {
+			printf("error: select timeout\n");
+			goto display_cleanup;
+		}
+        process_ds0_image(vdev, format.width, format.height);
     }
-exit:
-    printf("%s ==========release \n", __func__);
+
+display_cleanup:
     mtx.lock();
-    capture.release();
+    video_stop(vdev);
+	video_cleanup(vdev);
     mtx.unlock();
 }