examples: add code to demostrate how to use Fraunhofer H266/VVC encod…

…er & Intel AV1 encoder

.gitignore

@@ -65,3 +65,10 @@ cdeosplayer/kantv/src/main/assets/deepspeech-0.9.3-models.tflite
 
 #generated by source code,so skip it
 cdeosplayer/kantv/src/main/jniLibs/arm64-v8a/libopenblas.so
+
+prebuilts/toolchain/android-ndk-r26c-linux.zip
+prebuilts/toolchain/android-ndk-r26c/
+prebuilts/toolchain/emsdk/
+prebuilts/toolchain/*.zip
+prebuilts/toolchain/*.xz
+prebuilts/toolchain/*.gz

README.md

@@ -53,6 +53,7 @@ sudo apt-get update
 sudo apt-get install build-essential -y
 sudo apt-get install cmake -y
 sudo apt-get install curl -y
+sudo apt-get install wget -y
 sudo apt-get install python -y
 sudo apt-get install tcl expect -y
 sudo apt-get install nginx -y
@@ -83,6 +84,15 @@ pip3 install meson ninja
 
 echo "export PATH=/home/`whoami`/.local/bin:\$PATH" >> ~/.bashrc
 
+```
+
+or run below script accordingly after fetch project's source code
+
+```
+
+./build/prebuild.sh
+
+
 ```
 
 - Android NDK & Android Studio
@@ -145,7 +155,7 @@ git checkout master
 
 #### Build native codes
 
-modify <a href="https://github.com/cdeos/kantv/blob/master/build/envsetup.sh#L59">build/envsetup.sh</a> accordingly before launch build
+modify <a href="https://github.com/cdeos/kantv/blob/master/build/envsetup.sh#L85">build/envsetup.sh</a> accordingly before launch build
 
 pay attention <a href="https://github.com/cdeos/kantv/blob/master/external/whispercpp/CMakeLists.txt#L54">here and modify it accordingly</a> if build-target is kantv-android and running Android device is NOT Xiaomi 14
 
@@ -154,6 +164,9 @@ a VERY powerful Linux PC / Linux workstation is HIGHLY recommended for this step
 ```
 . build/envsetup.sh
 
+(download android-ndk-r26c to prebuilts/toolchain, skip this step if android-ndk-r26c is already exist)
+./build/prebuild-download.sh
+
 ```
 ![Screenshot from 2024-03-21 21-41-41](https://github.com/zhouwg/kantv/assets/6889919/3e13946f-596b-44be-9716-5793ce0c7263)
 

build/envsetup.sh

@@ -52,11 +52,15 @@ export PROJECT_BRANCH=`git branch | grep "*" | cut -f 2 -d ' ' `
 export PROJECT_ROOT_PATH=${PROJECT_HOME_PATH}
 export PROJECT_OUT_PATH=${PROJECT_ROOT_PATH}/out
 export FF_PREFIX=${PROJECT_OUT_PATH}/${BUILD_TARGET}/
+#export KANTV_TOOLCHAIN_PATH=/opt/kantv-toolchain
+export KANTV_TOOLCHAIN_PATH=${PROJECT_ROOT_PATH}/prebuilts/toolchain
+export LOCAL_WHISPERCPP_PATH=${PROJECT_ROOT_PATH}/external/whispercpp
+
+
+export KANTV_PROJECTS="kantv-linux kantv-android kantv-ios kantv-wasm"
+export KANTV_PROJECTS_DEBUG="kantv-android-debug"
 
 
-#modify following lines to adapt to local dev envs
-#export KANTV_TOOLCHAIN_PATH=${PROJECT_ROOT_PATH}/toolchain
-export KANTV_TOOLCHAIN_PATH=/opt/kantv-toolchain
 #API21：Android 5.0 (Android L)Lollipop
 #API22：Android 5.1 (Android L)Lollipop
 #API23：Android 6.0 (Android M)Marshmallow
@@ -77,12 +81,9 @@ export ANDROID_PLATFORM=android-34
 #export ANDROID_NDK=${KANTV_TOOLCHAIN_PATH}/android-ndk-r24
 export ANDROID_NDK=${KANTV_TOOLCHAIN_PATH}/android-ndk-r26c
 
-export LOCAL_WHISPERCPP_PATH=${PROJECT_ROOT_PATH}/external/whispercpp
-export UPSTREAM_WHISPERCPP_PATH=~/cdeos/whisper.cpp
-
-export KANTV_PROJECTS="kantv-linux kantv-android kantv-ios kantv-wasm"
-export KANTV_PROJECTS_DEBUG="kantv-android-debug"
 
+#modify following lines to adapt to local dev envs
+export UPSTREAM_WHISPERCPP_PATH=~/whisper.cpp
 
 
 . ${PROJECT_ROOT_PATH}/build/public.sh || (echo "can't find public.sh"; exit 1)

build/prebuild-download.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2024- KanTV Authors
+
+# Description: download Android NDK for build project
+
+set -e
+
+if [ "x${PROJECT_ROOT_PATH}" == "x" ]; then
+    echo "pwd is `pwd`"
+    echo "pls run . build/envsetup in project's toplevel directory firstly"
+    exit 1
+fi
+
+. ${PROJECT_ROOT_PATH}/build/public.sh || (echo "can't find public.sh"; exit 1)
+
+show_pwd
+
+echo "ANDROID_NDK: ${ANDROID_NDK}"
+
+is_android_ndk_exist=1
+
+if [ ! -d ${ANDROID_NDK} ]; then
+    echo -e "${TEXT_RED}NDK ${ANDROID_NDK} not exist, pls check...${TEXT_RESET}\n"
+    is_android_ndk_exist=0
+fi
+
+if [ ! -f ${ANDROID_NDK}/build/cmake/android.toolchain.cmake ]; then
+    echo -e "${TEXT_RED}NDK ${ANDROID_NDK} not exist, pls check...${TEXT_RESET}\n"
+    is_android_ndk_exist=0
+fi
+
+if [ ${is_android_ndk_exist} -eq 0 ]; then
+    echo -e "begin downloading android ndk \n"
+    wget --no-config --quiet --show-progress -O ${ANDROID_NDK}-linux.zip  https://dl.google.com/android/repository/android-ndk-r26c-linux.zip
+    cd ${PROJECT_ROOT_PATH}/prebuilts/toolchain
+    unzip android-ndk-r26c-linux.zip
+    cd ${PROJECT_ROOT_PATH}
+
+    if [ $? -ne 0 ]; then
+        printf "failed to download android ndk to %s \n" "${ANDROID_NDK}"
+        exit 1
+    fi
+
+    printf "android ndk saved to ${ANDROID_NDK} \n\n"
+else
+    printf "android ndk already exist:${ANDROID_NDK} \n\n"
+fi

build/prebuild.sh

@@ -12,6 +12,7 @@ sudo apt-get update
 sudo apt-get install build-essential -y
 sudo apt-get install cmake -y
 sudo apt-get install curl -y
+sudo apt-get install wget -y
 sudo apt-get install python -y
 sudo apt-get install tcl expect -y
 sudo apt-get install nginx -y

docs/FAQ.md

@@ -16,12 +16,16 @@ Project KanTV was almost/completely done by myself since 05-2021 and NO IPR conc
 
 <hr>
 
-- What's the relationship between kantv and whisper.cpp
+- What's the relationship between kantv and whisper.cpp/GGML
 
+Project KanTV has NO or NO personal relationship with whisper.cpp/GGML but only use source code of whisper.cpp as ASR engine.
 
-Project KanTV has NO or NO personal relationship with whisper.cpp but only use source code of whisper.cpp as ASR engine.
+I heard whisper.cpp on Jan 2024 at the first time and just use it generate subtitle for original English short video which referenced in Chinese articles on personal WeChat public account, but I didn’t read the source code of whisper.cpp at that time. sadly, my personal WeChat public account was forcefully closed on 01-28-2024. after that, I [decided to migrate some personal software project to Github since 02-22-2024](https://github.com/zhouwg/kantv/blob/master/release/README.md#L122). I have to say that I really spent much time/efforts on personal WeChat public account for public interests or my country's interests before 02-22-2024.
+
+I have to say that life is like a box of chocolate, you never know what you’re going to get. I decided to start integrating the great whisper.cpp to this project since 03-05-2024 and I did it on 03-16-2024 finally. I have to say that I was touched with and motivated by the great open-source project whisper.cpp from 03-05-2024 to 03-22-2024.
+
+At the same time, I have to say that GGML's whisper.cpp is a real excellent and amazing open source AI project and very helpful for C/C++ programmer and the original author of GGML is the only person I know of who is AI expert and modern C++ master and familiar with both iOS(app / native ) and Android(app / native) and Linux(app / native) software development at the same time(I know a few programmers who are familiar with both iOS(app / dev) and Android (app / native) and Linux(app / native) software development but they also know very little about real AI tech) and I have to say that the original author of GGML made a huge contribution to our planet.
 
-But, I have to say that GGML's whisper.cpp is a real excellent and amazing open source AI project and very helpful for C/C++ programmer and the original author of GGML is the only person I know of who is AI expert and modern C++ master and familiar with both iOS(app / native ) and Android(app / native) and Linux(app / native) software development at the same time(I know a few programmers who are familiar with both iOS(app / dev) and Android (app / native) and Linux(app / native) software development but they also know very little about real AI tech) and I have to say that the original author of GGML made a huge contribution to our planet.
 
 
 <hr>

docs/acknowledgement.md

@@ -4,23 +4,26 @@ Project KanTV has used/tried following open-source projects(list in here is not
 
 <ul>
      <li><a href="https://github.com/torvalds/linux"     target="_blank">Linux</a></li>
-     <li><a href="https://blog.google/products/android/" target="_blank" rel="noopener">Android</a></li>
+     <li><a href="https://blog.google/products/android/" target="_blank" rel="noopener">Google Android</a></li>
      <li><a href="http://ffmpeg.org/" target="_blank" rel="noopener">FFmpeg</a></li>
-     <li><a href="https://github.com/ggerganov/whisper.cpp" target="_blank" rel="noopener">whisper.cpp</a></li>
-     <li><a href="https://github.com/bilibili/ijkplayer" target="_blank" rel="noopener">ijkplayer</a></li>
+     <li><a href="https://github.com/ggerganov/whisper.cpp" target="_blank" rel="noopener">GGML whisper.cpp</a></li>
+     <li><a href="https://github.com/ggerganov/whisper.cpp" target="_blank" rel="noopener">GGML llama.cpp</a></li>
+     <li><a href="https://github.com/bilibili/ijkplayer" target="_blank" rel="noopener">Bilibili ijkplayer</a></li>
+     <li><a href="https://github.com/mozilla/DeepSpeech" target="_blank" rel="noopener">Mozilla DeepSpeech</a></li>
      <li><a href="https://www.videolan.org/vlc/" target="_blank" rel="noopener">libx264/libx265</a></li>
      <li><a href="https://github.com/deniskropp/DirectFB" target="_blank" rel="noopener">DirectFB</a></li>
      <li><a href="https://www.libsdl.org/" target="_blank" rel="noopener">SDL</a></li>
      <li><a href="https://gstreamer.freedesktop.org/" target="_blank" rel="noopener">GStreamer</a></li>
      <li><a href="https://github.com/mhroth/tinywav/" target="_blank" rel="noopener">tinywav</a></li>
-     <li><a href="https://www.intel.com/content/www/us/en/developer/articles/technical/scalable-video-technology.html" target="_blank" rel="noopener">SVT-AV1</a></li>
-     <li><a href="https://www.intel.com/content/www/us/en/developer/articles/technical/scalable-video-technology.html" target="_blank" rel="noopener">SVT-HEVC</a></li>
-     <li><a href="https://github.com/fraunhoferhhi/vvenc" target="_blank" rel="noopener">VVenc</a></li>
-     <li><a href="https://aomedia.org/" target="_blank" rel="noopener">AOM-AV1</a></li>
-     <li><a href="https://opencv.org/" target="_blank" rel="noopener">OpenCV</a></li>
+     <li><a href="https://www.intel.com/content/www/us/en/developer/articles/technical/scalable-video-technology.html" target="_blank" rel="noopener">Intel AV1</a></li>
+     <li><a href="https://www.intel.com/content/www/us/en/developer/articles/technical/scalable-video-technology.html" target="_blank" rel="noopener">Intel HEVC</a></li>
+     <li><a href="https://github.com/fraunhoferhhi/vvenc" target="_blank" rel="noopener">Fraunhofer H266/VVC encoder</a></li>
+     <li><a href="https://github.com/fraunhoferhhi/vvdec" target="_blank" rel="noopener">Fraunhofer H266/VVC decoder</a></li>
+     <li><a href="https://aomedia.org/" target="_blank" rel="noopener">AOM AV1</a></li>
+     <li><a href="https://opencv.org/" target="_blank" rel="noopener">Intel OpenCV</a></li>
      <li><a href="https://webrtc.github.io/webrtc-org/start/" target="_blank" rel="noopener">WebRTC</a></li>
-     <li><a href="https://github.com/Tencent/ncnn" target="_blank" rel="noopener">Tencent/ncnn</a></li>
-     <li><a href="https://github.com/shaka-project/shaka-packager" target="_blank" rel="noopener">ShakaPackager</a></li>
+     <li><a href="https://github.com/Tencent/ncnn" target="_blank" rel="noopener">Tencent ncnn</a></li>
+     <li><a href="https://github.com/shaka-project/shaka-packager" target="_blank" rel="noopener">Google ShakaPackager</a></li>
      <li><a href="https://github.com/ossrs/srs" target="_blank" rel="noopener">SRS</a></li>
      <li><a href="https://github.com/nihui/ruapu" target="_blank" rel="noopener">RUAPU</a></li>
      <li>......</li>

examples/ff_encode/Makefile

@@ -30,10 +30,10 @@ STRIP=$(CROSS)strip
 FF_SRC_PATH := ${PROJECT_ROOT_PATH}/external/ffmpeg/
 FF_LIB_PATH := ${PROJECT_ROOT_PATH}/out/${BUILD_TARGET}/lib/
 
-# common for x86,android,ios,wasm
-G_LDFLAGS    = -L${FF_LIB_PATH}  -lx264 -lx265 -laom -lwebp -lsharpyuv -lavformat -lavcodec -lx264 -lx265 -laom -lwebp -lsharpyuv -lvvdec -lSvtAv1Enc -lSvtAv1Dec -lvvenc -lswscale -lavutil -lswresample
+# common for linux,android,ios,wasm
+G_LDFLAGS    = -L${FF_LIB_PATH} -lavformat -lavcodec -lx264 -lx265 -laom -lwebp -lsharpyuv -lvvenc -lvvdec -lSvtAv1Enc -lSvtAv1Dec -lswscale -lavutil -lswresample
 
-# for x86
+# for linux
 G_LDFLAGS += -lSDL2 -lasound -lXv -lX11 -lXext -lxcb -lXfixes -lXinerama -pthread -lm -latomic -lz -ldl -lstdc++
 
 

examples/ff_encode/main.c

@@ -39,7 +39,8 @@
 #include "libavutil/vkey.h"
 
 
-#define STREAM_FRAME_RATE   5
+//attention: this value is depend on machine's performance heavily
+#define STREAM_FRAME_RATE   1
 
 #define STREAM_DURATION     10.0
 
@@ -48,6 +49,8 @@
 #define STREAM_PIX_FMT      AV_PIX_FMT_YUV420P /* default pix_fmt */
 
 static int                  g_pattern       = 0;
+static int                  g_codec         = 0;  //default codec is h264
+
 static struct SwsContext    *g_sws_context  = NULL;
 
 typedef struct OutputStream {
@@ -157,9 +160,8 @@ static void add_stream_by_codecid(OutputStream *ost, AVFormatContext *oc,
         ost->st->time_base = (AVRational){ 1, STREAM_FRAME_RATE };
         c->time_base       = ost->st->time_base;
 
-        c->gop_size      = 1;
-
-        c->pix_fmt       = STREAM_PIX_FMT;
+        c->gop_size        = 1;
+        c->pix_fmt         = STREAM_PIX_FMT;
 
         if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
             c->max_b_frames = 2;
@@ -245,14 +247,23 @@ static void add_stream_by_codecname(OutputStream *ost, AVFormatContext *oc,
         c->time_base       = ost->st->time_base;
 
         c->gop_size      = 1;
-        c->pix_fmt       = STREAM_PIX_FMT;
+
+        if (2 == g_codec) //make H266 encoder happy
+            c->pix_fmt   = AV_PIX_FMT_YUV420P10LE;
+        else
+            c->pix_fmt   = AV_PIX_FMT_YUV420P;
 
         if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
             c->max_b_frames = 2;
         }
+
         if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO) {
             c->mb_decision = 2;
         }
+
+        if (4 == g_codec) //make Intel-AV1 encoder happy
+            av_opt_set(c->priv_data, "crf", "35", 0);
+
         break;
 
     default:
@@ -520,7 +531,7 @@ static void ffmpeg_encoder_set_frame_yuv_from_rgb(uint8_t *rgb, int width, int h
 }
 
 
-uint8_t* generate_rgb(AVFrame *frame, int width, int height, int pts, uint8_t *rgb) {
+static uint8_t* generate_rgb(AVFrame *frame, int width, int height, int pts, uint8_t *rgb) {
     int x, y, cur;
     rgb = realloc(rgb, 3 * sizeof(uint8_t) * height * width);
     for (y = 0; y < height; y++) {
@@ -666,7 +677,7 @@ static void showUsage()
         "Options:\n" \
         " -n <filename>   base file name of encoded file\n" \
         " -f <format>     container name(ts/mp4, codec webp is only valid for ts currently)\n" \
-        " -c <0/1/2/3>    codec: 0-h264 1-h265 2-av1 3-webp\n" \
+        " -c <0/1/2/3/4>  codec: 0 - h264 1 - h265 2 - h266 3 - AOM AV1 4 - Intel AV1  5 - webp\n" \
         " -p <0/1>        pattern: 0 1\n" \
         " ?/h print usage infomation\n\n"
     );
@@ -725,6 +736,8 @@ int main(int argc, char **argv) {
 
             case 'c':
                 codecid  = atoi(optarg);
+                g_codec  = codecid;
+
                 LOGGD("codecid %d\n", codecid);
                 switch (codecid) {
                     case 0:
@@ -736,10 +749,18 @@ int main(int argc, char **argv) {
                         break;
 
                     case 2:
-                        codec_name = "libaom-av1";
+                        codec_name = "libvvenc";
                         break;
 
                     case 3:
+                        codec_name = "libaom-av1";
+                        break;
+
+                    case 4:
+                        codec_name = "libsvtav1";
+                        break;
+
+                    case 5:
                         codec_name = "libwebp";
                         break;
 

external/.gitignore

@@ -1,4 +1,3 @@
-FFmpeg/
 DeepSpeech/
 gstreamer/
 ncnn/
@@ -8,14 +7,9 @@ llamacpp/
 *.a
 *.so
 *.o
-config.log
-config.status
-configure~
-Makefile
-libpng-config
-libpng12-config
 
 whispercpp/bench
 whispercpp/main
 whispercpp/quantize
 
+ffmpeg-android-build/

external/ffmpeg-deps/.gitignore

@@ -1,13 +1,15 @@
-FFmpeg/
-DeepSpeech/
-gstreamer/
-ncnn/
-CLBlast/
-llamacpp/
-
 uavs3e/version.h
 uavs3d/version.h
 
+fribidi-1.0.13/libtool
+
+config.log
+config.status
+configure~
+Makefile
+libpng-config
+libpng12-config
 
+*.o
 *.a
 *.so