Update image colorization and video frame extraction demos to use BMF…

… convert backend utility function * remove unused import line causing import error * use bmf_convert * use bmf_convert prototype * update gitignore * prototype testing bmf_convert * ensure module ABI is compatible with BabitMF-GPU 0.0.9 * remove commented-out code * correctly gitignore folders * comment out use c++11 abi option * update deoldify demo jupyter notebook * update video frame extract jupyter notebook https://code.byted.org/videoarch/bmf/merge_requests/838
BabitMF · Sep 5, 2024 · 17aee9d · 17aee9d
1 parent e3f240d
commit 17aee9d
Show file tree

Hide file tree

Showing 7 changed files with 24 additions and 27 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 .idea/*
+.venv/
 *.mov
 *.json
 *.pyc
@@ -8,6 +9,7 @@
 .DS_Store
 *.o
 *.so
+*.sym
 /build*/
 !build.gradle
 bmf/c_module_sdk/build
@@ -21,7 +23,7 @@ bmf/engine/connector/cmake-build-debug
 bmf/go_module_sdk/.idea
 bmf/go_module_sdk/build
 bmf/go_module_sdk/cmake-build-debug
-
+output/
 
 .DS_Store
 */.DS_Store

diff --git a/bmf/demo/colorization_python/deoldify_demo.py b/bmf/demo/colorization_python/deoldify_demo.py
@@ -2,7 +2,6 @@
 
 # Add the DeOldify folder to the python search path
 import sys
-
 sys.path.insert(0, './DeOldify')
 print(sys.path)
 

diff --git a/bmf/demo/colorization_python/deoldify_demo_colab.ipynb b/bmf/demo/colorization_python/deoldify_demo_colab.ipynb
@@ -283,6 +283,7 @@
         "from bmf import ProcessResult, Packet, Timestamp, VideoFrame\n",
         "import PIL\n",
         "import bmf.hml.hmp as mp\n",
+        "from bmf.lib._bmf import sdk\n",
         "\n",
         "from deoldify import device\n",
         "from deoldify.device_id import DeviceId\n",
@@ -341,8 +342,8 @@
         "                if packet.timestamp != Timestamp.UNSET and packet.is_(VideoFrame):\n",
         "\n",
         "                    vf = packet.get(VideoFrame)\n",
-        "                    rgb = mp.PixelInfo(mp.kPF_RGB24)\n",
-        "                    np_vf = vf.reformat(rgb).frame().plane(0).numpy()\n",
+        "                    dst_md = sdk.MediaDesc().pixel_format(mp.kPF_RGB24)\n",
+        "                    np_vf = sdk.bmf_convert(vf, sdk.MediaDesc(), dst_md).frame().plane(0).numpy()\n",
         "\n",
         "                    # numpy to PIL\n",
         "                    image = Image.fromarray(np_vf.astype('uint8'), 'RGB')\n",
@@ -534,4 +535,4 @@
   },
   "nbformat": 4,
   "nbformat_minor": 0
-}
+}
diff --git a/bmf/demo/colorization_python/py_deoldify_module.py b/bmf/demo/colorization_python/py_deoldify_module.py
@@ -4,6 +4,8 @@
 import PIL
 import bmf.hml.hmp as mp
 
+from bmf.lib._bmf import sdk
+
 from deoldify import device
 from deoldify.device_id import DeviceId
 import torch
@@ -67,8 +69,8 @@ def process(self, task):
                         VideoFrame):
 
                     vf = packet.get(VideoFrame)
-                    rgb = mp.PixelInfo(mp.kPF_RGB24)
-                    np_vf = vf.reformat(rgb).frame().plane(0).numpy()
+                    dst_md = sdk.MediaDesc().pixel_format(mp.kPF_RGB24)
+                    np_vf = sdk.bmf_convert(vf, sdk.MediaDesc(), dst_md).frame().plane(0).numpy()
 
                     # numpy to PIL
                     image = Image.fromarray(np_vf.astype('uint8'), 'RGB')

diff --git a/bmf/demo/video_frame_extraction/CMakeLists.txt b/bmf/demo/video_frame_extraction/CMakeLists.txt
@@ -6,6 +6,9 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_FLAGS "-Wno-deprecated-declarations")
 set(CMAKE_INSTALL_PREFIX ${PROJECT_SOURCE_DIR})
 
+# You may need to add this option for BabitMF version 0.0.9 or lower to solve an ABI missing symbol issue in the output `libjpeg_encoder.so` dynamic library
+# add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
+
 if (NOT DEFINED CUDA_ROOT)
     set(CUDA_ROOT /usr/local/cuda)
 endif()

diff --git a/bmf/demo/video_frame_extraction/jpeg_encoder.cpp b/bmf/demo/video_frame_extraction/jpeg_encoder.cpp
@@ -22,6 +22,7 @@
 #include <bmf/sdk/video_frame.h>
 #include <bmf/sdk/ffmpeg_helper.h>
 #include <bmf/sdk/error_define.h>
+#include <bmf/sdk/convert_backend.h>
 
 extern "C" {
 #include <libavcodec/packet.h>
@@ -152,18 +153,10 @@ class jpeg_encoder : public bmf_sdk::Module {
                             << vf.frame().nplanes() << " planes.";
                     }
 
-                    if (!vf_rgb)
-                        vf_rgb = VideoFrame::make(width, height, RGB, kCUDA);
-                    Tensor t_img = vf_rgb.frame().plane(0);
-                    hmp::img::yuv_to_rgb(t_img, vf.frame().data(), NV12, kNHWC);
-                    if (!vf_yuv_from_rgb)
-                        vf_yuv_from_rgb =
-                            VideoFrame::make(width, height, H420, kCUDA);
-                    TensorList tl = vf_yuv_from_rgb.frame().data();
-                    hmp::img::rgb_to_yuv(tl, vf_rgb.frame().plane(0), H420,
-                                         kNHWC);
-
-                    vf_final_p = &vf_yuv_from_rgb;
+                    MediaDesc dp{};
+                    dp.pixel_format(hmp::PF_YUV420P);
+                    vf_rfmt = bmf_convert(vf, MediaDesc{}, dp);
+                    vf_final_p = &vf_rfmt;
                 } else {
                     BMFLOG_NODE(BMF_INFO, node_id_)
                         << "frame comes from cpu decoder.";

diff --git a/bmf/demo/video_frame_extraction/video_frame_extraction_acceleration.ipynb b/bmf/demo/video_frame_extraction/video_frame_extraction_acceleration.ipynb
@@ -732,6 +732,7 @@
         "#include <bmf/sdk/video_frame.h>\n",
         "#include <bmf/sdk/ffmpeg_helper.h>\n",
         "#include <bmf/sdk/error_define.h>\n",
+        "#include <bmf/sdk/convert_backend.h>\n",
         "\n",
         "/*\n",
         "extern \"C\" {\n",
@@ -853,14 +854,10 @@
         "                                                        << vf.frame().nplanes() << \" planes.\";\n",
         "                    }\n",
         "\n",
-        "                    if (!vf_rgb)\n",
-        "                        vf_rgb = VideoFrame::make(width, height, RGB, kCUDA);\n",
-        "                    Tensor t_img = vf_rgb.frame().plane(0);\n",
-        "                    hmp::img::yuv_to_rgb(t_img, vf.frame().data(), NV12, kNHWC);\n",
-        "                    if (!vf_yuv_from_rgb)\n",
-        "                        vf_yuv_from_rgb = VideoFrame::make(width, height, H420, kCUDA);\n",
-        "                    TensorList tl = vf_yuv_from_rgb.frame().data();\n",
-        "                    hmp::img::rgb_to_yuv(tl, vf_rgb.frame().plane(0), H420, kNHWC);\n",
+        "                    MediaDesc dp{};\n",
+        "                    dp.pixel_format(hmp::PF_YUV420P);\n",
+        "                    vf_rfmt = bmf_convert(vf, MediaDesc{}, dp);\n",
+        "                    vf_final_p = &vf_rfmt;\n",
         "\n",
         "                    vf_final_p = &vf_yuv_from_rgb;\n",
         "                } else {\n",