From b3c40e515928f87d9ae2e9948141d94bf7951cc4 Mon Sep 17 00:00:00 2001 From: bunny720 Date: Wed, 20 Jul 2022 21:16:46 +0800 Subject: [PATCH 1/5] Encode app free rc control --- package/encode_app/src/main.cpp | 214 ++++++++++++++++++++++++++------ 1 file changed, 178 insertions(+), 36 deletions(-) diff --git a/package/encode_app/src/main.cpp b/package/encode_app/src/main.cpp index f6d9efc..7db62e7 100755 --- a/package/encode_app/src/main.cpp +++ b/package/encode_app/src/main.cpp @@ -186,7 +186,10 @@ typedef struct int isp_enabled; int v4l2_enabled; uint32_t *repeat; + uint32_t *repeat_en; uint32_t *drop; + uint32_t *drop_en; + uint32_t *framerate_mod; int *set_ae; unsigned char *out_framerate; int video_enabled; @@ -598,6 +601,19 @@ static void *v4l2_output(void *arg) } } +uint32_t queue_frame(int channel) +{ + if((pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr & V4L2_REPEAT_FLAG) == 0) + { + enqueue_buf(pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr, channel); + } + pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr = V4L2_INVALID_INDEX; + pCtx->v4l2_rp[channel]++; + pCtx->v4l2_rp[channel] %= ISP_ADDR_BUFFER_CNT; + + return 0; +} + static void *encode_ch(void *arg) { // printf("%s\n", __FUNCTION__); @@ -611,7 +627,30 @@ static void *encode_ch(void *arg) int frame_size = stride*pCtx->Cfg[channel].height*3/2; uint32_t repeated = 0; - uint32_t droped = 0; + int32_t droped = 0; + uint32_t framerate_tail = 0; + uint32_t framerate_tail_cnt = 0; + uint32_t framerate_tail_gap = 0; + uint32_t in_framecnt = 0; + uint32_t out_framecnt = 0; + int tail_flag = 0; + int common_drop_flag = 0; + + if(pCtx->framerate_mod[channel]) + { + if(pCtx->drop_en[channel]) + { + framerate_tail = (pCtx->framerate[channel]/(pCtx->drop[channel]+1))%pCtx->out_framerate[channel]; + framerate_tail_cnt = framerate_tail; + framerate_tail_gap = (pCtx->framerate[channel]/framerate_tail) - 1; + } + else if(pCtx->repeat_en[channel]) + { + framerate_tail = pCtx->framerate_mod[channel]; + framerate_tail_cnt = framerate_tail; + framerate_tail_gap = pCtx->framerate[channel]/framerate_tail; + } + } pCtx->encoding[channel] = 1; while(pCtx->start) @@ -668,26 +707,73 @@ static void *encode_ch(void *arg) pCtx->start_time[channel] = time; } - if(pCtx->drop[channel]) + if(pCtx->drop_en[channel]) { - if(droped < pCtx->drop[channel]) + if(pCtx->framerate_mod[channel]) //with framerate tail { - if((pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr & V4L2_REPEAT_FLAG) == 0) + if((in_framecnt % (framerate_tail_gap+1) == 0) || tail_flag) { - enqueue_buf(pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr, channel); + tail_flag = 1; + queue_frame(channel); + in_framecnt++; + framerate_tail_cnt--; + if(in_framecnt >= pCtx->framerate[channel]) + { + in_framecnt = 0; + framerate_tail_cnt = framerate_tail; + } + droped++; + if(droped >= pCtx->drop[channel] + 1) + { + droped = 0; + tail_flag = 0; + } + continue; } - pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr = V4L2_INVALID_INDEX; - pCtx->v4l2_rp[channel]++; - pCtx->v4l2_rp[channel] %= ISP_ADDR_BUFFER_CNT; - droped++; - continue; + else if((pCtx->drop[channel] && (in_framecnt % (pCtx->drop[channel]+1) == 0)) || common_drop_flag) + { + common_drop_flag = 1; + queue_frame(channel); + in_framecnt++; + if(in_framecnt >= pCtx->framerate[channel]) + { + in_framecnt = 0; + framerate_tail_cnt = framerate_tail; + } + droped++; + if(droped >= pCtx->drop[channel]) + { + droped = 0; + common_drop_flag = 0; + } + continue; + } + else + { + in_framecnt++; + if(in_framecnt >= pCtx->framerate[channel]) + { + in_framecnt = 0; + framerate_tail_cnt = framerate_tail; + } + droped = 0; + } + } - else + else //without framerate tail { - droped = 0; + if(droped < pCtx->drop[channel]) + { + queue_frame(channel); + droped++; + continue; + } + else + { + droped = 0; + } } } - //set LTR if (pCtx->Cfg[channel].bEnableLTR && pCtx->nLTRFreq[channel] > 0) { @@ -734,33 +820,74 @@ static void *encode_ch(void *arg) if(pCtx->enable_v4l2[channel] == 1) { - if(pCtx->repeat[channel]) + if(pCtx->repeat_en[channel]) { - if(repeated < pCtx->repeat[channel]) + if(framerate_tail) // with framerate tail { - repeated++; + if(((in_framecnt % framerate_tail_gap) == 0) && framerate_tail_cnt) + { + + if(repeated < pCtx->repeat[channel] + 1) + { + repeated++; + } + else + { + repeated = 0; + framerate_tail_cnt--; + queue_frame(channel); + in_framecnt++; + if(in_framecnt >= pCtx->framerate[channel]) + { + in_framecnt = 0; + framerate_tail_cnt = framerate_tail; + } + + } + } + else + { + if(repeated < pCtx->repeat[channel]) + { + repeated++; + } + else + { + repeated = 0; + queue_frame(channel); + in_framecnt++; + if(in_framecnt >= pCtx->framerate[channel]) + { + in_framecnt = 0; + framerate_tail_cnt = framerate_tail; + } + } + } } - else + else //without framerate tail { - repeated = 0; - if((pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr & V4L2_REPEAT_FLAG) == 0) + if(repeated < pCtx->repeat[channel]) { - enqueue_buf(pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr, channel); + repeated++; } - pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr = V4L2_INVALID_INDEX; - pCtx->v4l2_rp[channel]++; - pCtx->v4l2_rp[channel] %= ISP_ADDR_BUFFER_CNT; + else + { + repeated = 0; + queue_frame(channel); + in_framecnt++; + in_framecnt %= pCtx->framerate[channel]; } + } } - else + else if(pCtx->drop_en[channel]) { - if((pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr & V4L2_REPEAT_FLAG) == 0) - { - enqueue_buf(pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr, channel); - } - pCtx->v4l2_rev[channel][pCtx->v4l2_rp[channel]].addr = V4L2_INVALID_INDEX; - pCtx->v4l2_rp[channel]++; - pCtx->v4l2_rp[channel] %= ISP_ADDR_BUFFER_CNT; + queue_frame(channel); + } + else if(!pCtx->drop_en[channel])// without repeate and drop + { + queue_frame(channel); + in_framecnt++; + in_framecnt %= pCtx->framerate[channel]; } } @@ -868,7 +995,9 @@ int free_context(void *arg) free(pCtx->encoding ); free(pCtx->ch ); free(pCtx->repeat ); + free(pCtx->repeat_en ); free(pCtx->drop ); + free(pCtx->drop_en ); free(pCtx->out_framerate ); free(pCtx->set_ae ); @@ -1795,7 +1924,10 @@ int alloc_context(void *arg) pCtx->encoding = (int*)malloc(sizeof(int) * pCtx->ch_cnt); pCtx->ch = (int*)malloc(sizeof(int) * pCtx->ch_cnt); pCtx->repeat = (uint32_t*)malloc(sizeof(uint32_t) * pCtx->ch_cnt); + pCtx->repeat_en = (uint32_t*)malloc(sizeof(uint32_t) * pCtx->ch_cnt); pCtx->drop = (uint32_t*)malloc(sizeof(uint32_t) * pCtx->ch_cnt); + pCtx->drop_en = (uint32_t*)malloc(sizeof(uint32_t) * pCtx->ch_cnt); + pCtx->framerate_mod = (uint32_t*)malloc(sizeof(uint32_t) * pCtx->ch_cnt); pCtx->out_framerate = (unsigned char*)malloc(sizeof(unsigned char) * pCtx->ch_cnt); pCtx->set_ae = (int*)malloc(sizeof(int) * pCtx->ch_cnt); @@ -2284,23 +2416,33 @@ int main(int argc, char *argv[]) DoEncRoi(pCtx->hEnc[i],pCtx->aryEncRoiCfg[i]); } - if((pCtx->out_framerate[i] % pCtx->framerate[i] == 0) || (pCtx->framerate[i] % pCtx->out_framerate[i] == 0)) + if((JPEG == pCtx->Cfg[i].profile) && (pCtx->out_framerate[i] != pCtx->framerate[i])) + { + pCtx->repeat_en[i] = 0; + pCtx->drop_en[i] = 0; + printf("WARNING: NOT SUPPORT RC CONTROL IN JPEG MODE, OUTPUT WITH INPUT FRAMERATE.\n"); + pCtx->out_framerate[i] = pCtx->framerate[i]; + } + else { if(pCtx->out_framerate[i] > pCtx->framerate[i]) { + pCtx->repeat_en[i] = 1; pCtx->repeat[i] = pCtx->out_framerate[i]/pCtx->framerate[i] - 1; printf("ch%d: repeat %d\n", i, pCtx->repeat[i]); + pCtx->framerate_mod[i] = pCtx->out_framerate[i]%pCtx->framerate[i]; + printf("ch%d: framerate_mod %d\n", i, pCtx->framerate_mod[i]); } else if(pCtx->out_framerate[i] < pCtx->framerate[i]) { + pCtx->drop_en[i] = 1; pCtx->drop[i] = pCtx->framerate[i]/pCtx->out_framerate[i] - 1; printf("ch%d: drop %d\n", i, pCtx->drop[i]); + pCtx->framerate_mod[i] = pCtx->framerate[i]%pCtx->out_framerate[i]; + // pCtx->framerate_mod[i] = pCtx->out_framerate[i]%pCtx->framerate[i]; + printf("ch%d: framerate_mod %d\n", i, pCtx->framerate_mod[i]); } } - else - { - printf("ch%d: Cannot handle -r: %d\n", i, pCtx->out_framerate[i]); - } } } } From b6e9e9c7f74b0f9428bb16625eb4a92be0023a2d Mon Sep 17 00:00:00 2001 From: bunny720 Date: Tue, 26 Jul 2022 16:30:48 +0800 Subject: [PATCH 2/5] cancel cpu2dsp task when terminate ffmpeg --- package/ffmpeg_canaan/0029-buildroot-ffmpeg.patch | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 package/ffmpeg_canaan/0029-buildroot-ffmpeg.patch diff --git a/package/ffmpeg_canaan/0029-buildroot-ffmpeg.patch b/package/ffmpeg_canaan/0029-buildroot-ffmpeg.patch new file mode 100644 index 0000000..ec89482 --- /dev/null +++ b/package/ffmpeg_canaan/0029-buildroot-ffmpeg.patch @@ -0,0 +1,14 @@ +Index: b/libavfilter/af_audio3a.c +=================================================================== +--- a/libavfilter/af_audio3a.c ++++ b/libavfilter/af_audio3a.c +@@ -646,6 +646,9 @@ static av_cold void uninit(AVFilterConte + printf("%s>Error!\n", __FUNCTION__); + } + } ++ ++ pthread_cancel(s->thread_1); ++ pthread_join(s->thread_1, NULL); + + close(s->fd_mailbox); + close(s->fd_share_memory); From 5976213323103c8980f21ac8aaa636896e7a3bf2 Mon Sep 17 00:00:00 2001 From: cuiyan Date: Fri, 29 Jul 2022 15:12:48 +0800 Subject: [PATCH 3/5] modify default output framerate --- package/encode_app/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/encode_app/src/main.cpp b/package/encode_app/src/main.cpp index c6d4d75..07e2957 100755 --- a/package/encode_app/src/main.cpp +++ b/package/encode_app/src/main.cpp @@ -2325,7 +2325,7 @@ int main(int argc, char *argv[]) pCtx->ch[i] = i; pCtx->Cfg[i].channel = i; if(!pCtx->framerate[i]) pCtx->framerate[i] = 30; - if(!pCtx->out_framerate[i]) pCtx->out_framerate[i] = 30; + if(!pCtx->out_framerate[i]) pCtx->out_framerate[i] = pCtx->framerate[i]; if(!pCtx->Cfg[i].width) pCtx->Cfg[i].width = 1920; if(!pCtx->Cfg[i].height) pCtx->Cfg[i].height = 1080; if(!pCtx->Cfg[i].BitRate) pCtx->Cfg[i].BitRate = 4000000; From 5ce705d57334b0175a490fc46ce62afe5f3626a9 Mon Sep 17 00:00:00 2001 From: cuiyan Date: Tue, 9 Aug 2022 20:43:11 +0800 Subject: [PATCH 4/5] add disableAE --- package/encode_app/src/main.cpp | 91 ++++++++------------------------- 1 file changed, 22 insertions(+), 69 deletions(-) diff --git a/package/encode_app/src/main.cpp b/package/encode_app/src/main.cpp index 439cac5..f706332 100755 --- a/package/encode_app/src/main.cpp +++ b/package/encode_app/src/main.cpp @@ -154,8 +154,6 @@ typedef struct long *shared_phyAddr; void **shared_vAddr; unsigned int *shared_size; - unsigned int *exp; - unsigned int *agc; int *stride; int *width; int *height; @@ -190,10 +188,9 @@ typedef struct uint32_t *drop; uint32_t *drop_en; uint32_t *framerate_mod; - int *set_ae; unsigned char *out_framerate; int video_enabled; - int ae_enable; + uint32_t *ae_disable; int setQos; /* audio */ @@ -462,19 +459,6 @@ static void enqueue_buf(unsigned char index, int channel) } } -static void set_ae(char* dev_name, int ae_enable) -{ - if((dev_name[10] >= '2') && (dev_name[10] <= '5') && (ae_enable & 0x1)) - { - mediactl_set_ae(ISP_F2K_PIPELINE); - } - else if((dev_name[10] >= '6') && (dev_name[10] <= '9') && (ae_enable & 0x1)) - { - mediactl_set_ae(ISP_R2K_PIPELINE); - } - return; -} - static void *v4l2_output(void *arg) { printf("%s\n", __FUNCTION__); @@ -506,8 +490,6 @@ static void *v4l2_output(void *arg) buf.memory = V4L2_MEMORY_USERPTR; res = ioctl(pCtx->fd_v4l2[channel], VIDIOC_DQBUF, &buf); - - set_ae(pCtx->dev_name[channel], pCtx->ae_enable); if (res < 0 || errno == EINTR) { @@ -974,8 +956,6 @@ int free_context(void *arg) free(pCtx->shared_phyAddr ); free(pCtx->shared_vAddr ); free(pCtx->shared_size ); - free(pCtx->exp ); - free(pCtx->agc ); free(pCtx->stride ); free(pCtx->width ); free(pCtx->height ); @@ -1000,7 +980,7 @@ int free_context(void *arg) free(pCtx->drop ); free(pCtx->drop_en ); free(pCtx->out_framerate ); - free(pCtx->set_ae ); + free(pCtx->ae_disable ); return 0; } @@ -1241,17 +1221,6 @@ static int init_isp() pCtx->fd_isp = isp_video(&ds1_info, sensor_type, lcd_type); - if(pCtx->exp[0] > 0) - { - printf("isp exp = %d\n", pCtx->exp[0]); - video_set_ae_dgain_cfg(pCtx->exp[0]); - } - if(pCtx->agc[0] > 0) - { - printf("isp agc = %d\n", pCtx->agc[0]); - video_set_ae_again_cfg(pCtx->agc[0]); - } - printf("%s>fd_isp: 0x%x\n", __FUNCTION__, pCtx->fd_isp); sleep(1); //It seems ISP output is not stable at the beginning @@ -1309,6 +1278,7 @@ int init_v4l2() struct v4l2_format fmt; int i; int f2k=0, r2k=0; + int ae_disabled[2] = {0}; for(int j = 0; j < pCtx->ch_cnt; j++) { @@ -1324,6 +1294,21 @@ int init_v4l2() sem_init(&pCtx->pSemGetData[i],0,0); } + for(i = 0; i < pCtx->ch_cnt; i++) + { + if(pCtx->ae_disable[i]) + { + if((pCtx->dev_name[i][10] >= '2') && (pCtx->dev_name[i][10] <= '5') && (!ae_disabled[0])) + { + mediactl_disable_ae(ISP_F2K_PIPELINE); + } + else if((pCtx->dev_name[i][10] >= '6') && (pCtx->dev_name[i][10] <= '9') && (!ae_disabled[1])) + { + mediactl_disable_ae(ISP_R2K_PIPELINE); + } + } + } + if(mediactl_init(REAL_CONF_FILENAME, &(pCtx->dev_info[0])) < 0) { printf("mediactl_init error!\n"); @@ -1905,8 +1890,6 @@ int alloc_context(void *arg) pCtx->shared_phyAddr = (long*)malloc(sizeof(long) * pCtx->ch_cnt); pCtx->shared_vAddr = (void**)malloc(sizeof(void*) * pCtx->ch_cnt); pCtx->shared_size = (unsigned int*)malloc(sizeof(unsigned int) * pCtx->ch_cnt); - pCtx->exp = (unsigned int*)malloc(sizeof(unsigned int) * pCtx->ch_cnt); - pCtx->agc = (unsigned int*)malloc(sizeof(unsigned int) * pCtx->ch_cnt); pCtx->stride = (int*)malloc(sizeof(int) * pCtx->ch_cnt); pCtx->width = (int*)malloc(sizeof(int) * pCtx->ch_cnt); pCtx->height = (int*)malloc(sizeof(int) * pCtx->ch_cnt); @@ -1931,8 +1914,8 @@ int alloc_context(void *arg) pCtx->drop = (uint32_t*)malloc(sizeof(uint32_t) * pCtx->ch_cnt); pCtx->drop_en = (uint32_t*)malloc(sizeof(uint32_t) * pCtx->ch_cnt); pCtx->framerate_mod = (uint32_t*)malloc(sizeof(uint32_t) * pCtx->ch_cnt); + pCtx->ae_disable = (uint32_t*)malloc(sizeof(uint32_t) * pCtx->ch_cnt); pCtx->out_framerate = (unsigned char*)malloc(sizeof(unsigned char) * pCtx->ch_cnt); - pCtx->set_ae = (int*)malloc(sizeof(int) * pCtx->ch_cnt); memset(pCtx->Cfg,0,sizeof(EncSettings)); @@ -1957,8 +1940,6 @@ int parse_cmd(int argc, char *argv[]) printf("-o: output file name or rtsp\n"); printf("-w: width\n"); printf("-h: height\n"); - printf("-e: set sensor exposure rate\n"); - printf("-ag: analog gain\n"); printf("-fps: sensor input framerate\n"); printf("-r: encoder output framrate\n"); printf("-inframes: input frames for input file\n"); @@ -1976,7 +1957,7 @@ int parse_cmd(int argc, char *argv[]) printf("-GDRMode: GDR mode 0:GDR_VERTICAL 1:GDR_HORIZONTAL\n"); printf("-enableLTR: enbale long term reference picture and specifies LTR refresh frequency in number of frames,0 to disable use refresh frequency\n"); printf("-roi: roi config file\n"); - printf("-ae: enable ae\n"); + printf("-disableAE: disable ae\n"); printf("-conf: v4l2 config file\n"); /* audio */ printf("-alsa: enable audio\n"); @@ -1984,7 +1965,6 @@ int parse_cmd(int argc, char *argv[]) printf("-ar: audio sample rate\n"); printf("-af: auido sample format\n"); printf("-ad: audio device"); - printf("-lossless: enable jpeg lossless encode"); // printf("-aof: audio output frames\n"); return 1; } @@ -2077,27 +2057,6 @@ int parse_cmd(int argc, char *argv[]) pCtx->Cfg[cur_ch].height = atoi(argv[i+1]); printf("height %d\n", pCtx->Cfg[cur_ch].height); } - else if(strcmp(argv[i], "-e") == 0) - { - pCtx->exp[cur_ch] = atoi(argv[i+1]); - printf("exp = %d\n", pCtx->exp[cur_ch]); - if(pCtx->exp[cur_ch] <= 0 || pCtx->exp[cur_ch] > 128) - { - printf("wrong exp = %d\n", pCtx->exp[cur_ch]); - return -1; - } - pCtx->exp[cur_ch] *= 8; - } - else if(strcmp(argv[i], "-ag") == 0) - { - pCtx->agc[cur_ch] = atoi(argv[i+1]); - printf("agc = %d\n", pCtx->agc[cur_ch]); - if(pCtx->agc[cur_ch] < 0 || pCtx->agc[cur_ch] > 232) - { - printf("wrong agc = %d\n", pCtx->agc[cur_ch]); - return -1; - } - } else if(strcmp(argv[i], "-fps") == 0) { pCtx->framerate[cur_ch] = atoi(argv[i+1]); @@ -2167,12 +2126,6 @@ int parse_cmd(int argc, char *argv[]) pCtx->Cfg[cur_ch].SliceQP = nqp; printf("sliceqp %d\n", nqp); } - else if (strcmp(argv[i],"-lossless") == 0 ) - { - int lossless = atoi(argv[i+1]); - pCtx->Cfg[cur_ch].lossless = lossless; - printf("lossless %d\n", lossless); - } else if (strcmp(argv[i],"-minqp") == 0 ) { int nqp = atoi(argv[i+1]); @@ -2240,9 +2193,9 @@ int parse_cmd(int argc, char *argv[]) printf("roi_parse_conf ok\n"); } } - else if(strcmp(argv[i], "-ae") == 0) + else if(strcmp(argv[i], "-disableAE") == 0) { - pCtx->ae_enable = atoi(argv[i+1]); + pCtx->ae_disable[cur_ch] = atoi(argv[i+1]); } else if(strcmp(argv[i], "-conf") == 0) { From 5e2ff3ccba7d07b3f60855b85b6b743d55aa0d86 Mon Sep 17 00:00:00 2001 From: cuiyan Date: Wed, 10 Aug 2022 08:54:53 +0800 Subject: [PATCH 5/5] add ae flag --- package/encode_app/src/main.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package/encode_app/src/main.cpp b/package/encode_app/src/main.cpp index f706332..f6aacc7 100755 --- a/package/encode_app/src/main.cpp +++ b/package/encode_app/src/main.cpp @@ -1301,10 +1301,12 @@ int init_v4l2() if((pCtx->dev_name[i][10] >= '2') && (pCtx->dev_name[i][10] <= '5') && (!ae_disabled[0])) { mediactl_disable_ae(ISP_F2K_PIPELINE); + ae_disabled[0] = 1; } else if((pCtx->dev_name[i][10] >= '6') && (pCtx->dev_name[i][10] <= '9') && (!ae_disabled[1])) { mediactl_disable_ae(ISP_R2K_PIPELINE); + ae_disabled[1] = 1; } } }