From 5cd3a646f871f3f0604f4bbb1f00fcc51bbfc151 Mon Sep 17 00:00:00 2001 From: HZ1 Date: Thu, 19 May 2022 20:46:22 +0800 Subject: [PATCH] [Feature & BugFix]: isp-tuningd (#47) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update nncase to v1.6 and sync dev to main (#5) * add github actions (#2) * modify lic (#3) * nncase update to 1.6 (#4) Co-authored-by: HZ1 Co-authored-by: wangjianxin-canaan <102512117+wangjianxin-canaan@users.noreply.github.com> * fix: #46, add: isp-tuningd with jpeg hardware encoder Signed-off-by: 黄子懿 * add package isp-tuningd Signed-off-by: 黄子懿 * add isp-tuningd to default defconfig Signed-off-by: 黄子懿 * revert libvenc.so Signed-off-by: 黄子懿 * fix #51, free buffer for stdin callback when pause Signed-off-by: 黄子懿 * Update isp-tuningd.c Co-authored-by: Wentao Wu Co-authored-by: wangjianxin-canaan <102512117+wangjianxin-canaan@users.noreply.github.com> Co-authored-by: wycwyhwyq <5f20.6d9b@gmail.com> --- Config.in | 1 + configs/k510_crb_lp3_v0_1_defconfig | 1 + configs/k510_crb_lp3_v1_2_defconfig | 1 + package/isp-tuningd/Config.in | 8 + package/isp-tuningd/isp-tuningd.mk | 22 ++ package/isp-tuningd/src/Makefile | 12 + .../src}/isp-tuningd.c | 213 ++++++++++++++---- package/mediactl_lib/Config.in | 1 - package/mediactl_lib/mediactl_lib.mk | 1 - .../src/v4l2_drm_isptool/Makefile | 6 - 10 files changed, 209 insertions(+), 57 deletions(-) create mode 100644 package/isp-tuningd/Config.in create mode 100644 package/isp-tuningd/isp-tuningd.mk create mode 100644 package/isp-tuningd/src/Makefile rename package/{mediactl_lib/src/v4l2_drm_isptool => isp-tuningd/src}/isp-tuningd.c (74%) diff --git a/Config.in b/Config.in index db2d053..2277730 100644 --- a/Config.in +++ b/Config.in @@ -17,6 +17,7 @@ source "$BR2_EXTERNAL_K510_PATH/package/alsa_demo/Config.in" source "$BR2_EXTERNAL_K510_PATH/package/ffmpeg_canaan/Config.in" source "$BR2_EXTERNAL_K510_PATH/package/audio3a_lib/Config.in" source "$BR2_EXTERNAL_K510_PATH/package/mediactl_lib/Config.in" +source "$BR2_EXTERNAL_K510_PATH/package/isp-tuningd/Config.in" source "$BR2_EXTERNAL_K510_PATH/package/mailbox_demo/Config.in" source "$BR2_EXTERNAL_K510_PATH/package/dsp_scheduler/Config.in" source "$BR2_EXTERNAL_K510_PATH/package/live555_canaan/Config.in" diff --git a/configs/k510_crb_lp3_v0_1_defconfig b/configs/k510_crb_lp3_v0_1_defconfig index 35ab978..0e97e63 100644 --- a/configs/k510_crb_lp3_v0_1_defconfig +++ b/configs/k510_crb_lp3_v0_1_defconfig @@ -114,6 +114,7 @@ BR2_PACKAGE_LIBOPENSSL=y BR2_PACKAGE_FFMPEG_CANAAN=y BR2_PACKAGE_AUDIO3A_LIB=y BR2_PACKAGE_MEDIACTL_LIB=y +BR2_PACKAGE_ISP_TUNINGD=y BR2_PACKAGE_MAILBOX_DEMO=y BR2_PACKAGE_DSP_SCHEDULER=y BR2_PACKAGE_LIVE555_CANAAN=y diff --git a/configs/k510_crb_lp3_v1_2_defconfig b/configs/k510_crb_lp3_v1_2_defconfig index f573c1e..e5ef8f0 100644 --- a/configs/k510_crb_lp3_v1_2_defconfig +++ b/configs/k510_crb_lp3_v1_2_defconfig @@ -111,6 +111,7 @@ BR2_PACKAGE_ALSA_DEMO=y BR2_PACKAGE_FFMPEG_CANAAN=y BR2_PACKAGE_AUDIO3A_LIB=y BR2_PACKAGE_MEDIACTL_LIB=y +BR2_PACKAGE_ISP_TUNINGD=y BR2_PACKAGE_MAILBOX_DEMO=y BR2_PACKAGE_DSP_SCHEDULER=y BR2_PACKAGE_LIVE555_CANAAN=y diff --git a/package/isp-tuningd/Config.in b/package/isp-tuningd/Config.in new file mode 100644 index 0000000..cbbe456 --- /dev/null +++ b/package/isp-tuningd/Config.in @@ -0,0 +1,8 @@ + +config BR2_PACKAGE_ISP_TUNINGD + bool "isp-tuningd" + depends on BR2_riscv + select BR2_PACKAGE_VENC_LIB + select BR2_PACKAGE_LIBUV + help + k510 ISP tuning daemon diff --git a/package/isp-tuningd/isp-tuningd.mk b/package/isp-tuningd/isp-tuningd.mk new file mode 100644 index 0000000..3855a2f --- /dev/null +++ b/package/isp-tuningd/isp-tuningd.mk @@ -0,0 +1,22 @@ +################################################################################ +# +# isp-tuningd +# +################################################################################ +ISP_TUNINGD_LOCAL_PATH:= $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +ISP_TUNINGD_DIR_NAME := isp-tuningd +ISP_TUNINGD_APP_NAME := isp-tuningd + +ISP_TUNINGD_SITE = $(ISP_TUNINGD_LOCAL_PATH)/src +ISP_TUNINGD_SITE_METHOD = local + + +define ISP_TUNINGD_BUILD_CMDS + $(TARGET_MAKE_ENV) $(MAKE) -C $(@D) +endef + +define ISP_TUNINGD_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0755 $(@D)/$(ISP_TUNINGD_DIR_NAME) $(TARGET_DIR)/app/mediactl_lib/$(ISP_TUNINGD_APP_NAME) +endef + +$(eval $(generic-package)) diff --git a/package/isp-tuningd/src/Makefile b/package/isp-tuningd/src/Makefile new file mode 100644 index 0000000..9ba81a9 --- /dev/null +++ b/package/isp-tuningd/src/Makefile @@ -0,0 +1,12 @@ +CROSS_COMPILE = riscv64-linux- +CC=$(CROSS_COMPILE)gcc + +TARGET = isp-tuningd +OBJS = isp-tuningd.c + +all: + ${CC} ${OBJS} -Wall -lpthread -luv -lvenc -g -o ${TARGET} + @echo "Compile done." +clean: + @rm -f *.o ${TARGET} + @echo "Clean done." diff --git a/package/mediactl_lib/src/v4l2_drm_isptool/isp-tuningd.c b/package/isp-tuningd/src/isp-tuningd.c similarity index 74% rename from package/mediactl_lib/src/v4l2_drm_isptool/isp-tuningd.c rename to package/isp-tuningd/src/isp-tuningd.c index 7676eab..ad5188e 100644 --- a/package/mediactl_lib/src/v4l2_drm_isptool/isp-tuningd.c +++ b/package/isp-tuningd/src/isp-tuningd.c @@ -24,6 +24,9 @@ */ #include "uv/unix.h" +#include +#include +#include #include #include #include @@ -31,7 +34,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -40,6 +45,13 @@ #include #include #include +#include "enc_interface.h" + +#define SHARE_MEMORY_ALLOC _IOWR('m', 1, unsigned long) +#define SHARE_MEMORY_ALIGN_ALLOC _IOWR('m', 2, unsigned long) +#define SHARE_MEMORY_FREE _IOWR('m', 3, unsigned long) +#define SHARE_MEMORY_SHOW _IOWR('m', 4, unsigned long) + #define max(a,b) (a>b?a:b) #define REGISTER_BASE_ADDR 0x92630000 @@ -53,12 +65,47 @@ #define ISP_TOF_ADDR_END (ISP_TOF_ADDR_START + 0x0400 + 0x0324) #define REGISTER_END_ADDR (0x0324 + 0x0400 + 0x92670000) #define DEFAULT_PORT 9982 -#define PIC_YUV_SIZE 3133440 +#define PIC_YUV_WIDTH 1088 +#define PIC_YUV_HEIGHT 1920 +#define PIC_YUV_SIZE (PIC_YUV_WIDTH * PIC_YUV_HEIGHT * 3 / 2) int page_size = 4096; unsigned offset_in_page = 0; unsigned map_size = 1048576; void* map_base = MAP_FAILED; +EncoderHandle* henc = NULL; +uint8_t pic_write_buffer_select = 0; +uint8_t pic_write_buffer[2][PIC_YUV_SIZE + 6] = { + {0x99, 0x9a, PIC_YUV_SIZE & 0xff, (PIC_YUV_SIZE >> 8) & 0xff, + (PIC_YUV_SIZE >> 16) & 0xff, (PIC_YUV_SIZE >> 24) & 0xff}, + {0x99, 0x9a, PIC_YUV_SIZE & 0xff, (PIC_YUV_SIZE >> 8) & 0xff, + (PIC_YUV_SIZE >> 16) & 0xff, (PIC_YUV_SIZE >> 24) & 0xff}, +}; +uint8_t* mem_yuv_logic_addr = MAP_FAILED; +unsigned int mem_yuv_map_size = 0; +unsigned int mem_yuv_phy_addr = 0; + +struct share_memory_alloc_align_args { + unsigned int size; + unsigned int alignment; + unsigned int phyAddr; +}; + +int alloc_memory(int fd_share_memory, unsigned int size, unsigned int *phy_addr, unsigned int* actual_size) { + struct share_memory_alloc_align_args allocAlignMem; + + allocAlignMem.size = (size + 0xfff) & (~0xfff); + allocAlignMem.alignment = 4096; // align 4k for mmap + allocAlignMem.phyAddr = 0; + + if (ioctl(fd_share_memory, SHARE_MEMORY_ALIGN_ALLOC, &allocAlignMem) < 0) { + perror("failed to allocate from /dev/mem"); + return -1; + } + *phy_addr = allocAlignMem.phyAddr; + *actual_size = allocAlignMem.size; + return 0; +} uint32_t read_register(uint32_t addr) { if (addr < REGISTER_BASE_ADDR || addr > REGISTER_END_ADDR) { @@ -252,18 +299,46 @@ void on_open(uv_fs_t* open_req) { } } +void on_encode_jpeg_done(void) { + printf("encode jpeg done.\n"); + EncOutputStream output; + VideoEncoder_GetStream(henc, &output); + uint32_t buf_size = output.bufSize; + uint8_t buffer[buf_size + 6]; + const uint8_t buffer_header[6] = { + 0x99, + 0x31, + buf_size & 0xff, + (buf_size >> 8) & 0xff, + (buf_size >> 16) & 0xff, + (buf_size >> 24) & 0xff + }; + memcpy(buffer, buffer_header, 6); + memcpy(buffer + 6, output.bufAddr, buf_size); + VideoEncoder_ReleaseStream(henc, &output); + broadcast(buffer, sizeof(buffer), 0); +} + +struct file_data { + uint32_t ser; + uint32_t path_len; + uint32_t file_len; + uint8_t data[]; +}; + #define MESSAGE_HEAD 0x99 #define CMD_READ 0x34 #define CMD_WRITE 0x12 #define CMD_PAUSE 0x9A #define CMD_FILE 0xBC +#define CMD_JPEG 0x31 uint8_t state = 0; uint8_t* data; uint32_t data_cap = 0; uint32_t data_itr = 0; uint32_t data_len = 0; uint8_t flag_send_pic = 1; -void exec_cmd() { +void exec_cmd(void) { uint8_t cmd = *data; uint8_t* raw = data + 5; uint32_t raw_len = data_len - 5; @@ -283,6 +358,7 @@ void exec_cmd() { data_buffer[i * 2 + 1] = read_register(address); } broadcast(write_buffer, write_len, 0); + } else if (cmd == CMD_WRITE) { uint32_t block_num = raw_len / 8; const uint32_t write_len = 6 + block_num * 2 * 4; @@ -313,12 +389,7 @@ void exec_cmd() { fprintf(stderr, "Wrong data (%d) for pause command\n", *raw); } } else if (cmd == CMD_FILE) { - struct { - uint32_t ser; - uint32_t path_len; - uint32_t file_len; - uint8_t data[]; - }* file = raw; + struct file_data* file = (struct file_data*)raw; char path[file->path_len + 1]; path[file->path_len] = '\0'; memcpy(path, file->data, file->path_len); @@ -328,6 +399,22 @@ void exec_cmd() { req->uv_buf = uv_buf_init(req->buffer, file->file_len); printf("save file to %s, length: %d\n", path, file->file_len); uv_fs_open(uv_default_loop(), &req->open_req, path, O_CREAT | O_WRONLY | O_TRUNC, 0, on_open); + } else if (cmd == CMD_JPEG) { + // encode JPEG + uint8_t *pic_nv12_buffer = + pic_write_buffer[pic_write_buffer_select ^ 1] + 6; + memcpy(mem_yuv_logic_addr, pic_nv12_buffer, PIC_YUV_SIZE); + EncInputFrame frame = { + .width = PIC_YUV_WIDTH, + .height = PIC_YUV_HEIGHT, + .stride = (PIC_YUV_WIDTH + 0x1F) & (~0x1F), + .data = (unsigned char *)mem_yuv_phy_addr + }; + // FIXME: async + printf("encode jpeg start...\n"); + // VideoEncoder_EncodeOneFrame_Async(henc, &frame, on_encode_jpeg_done); + VideoEncoder_EncodeOneFrame(henc, &frame); + on_encode_jpeg_done(); } else { // unknown cmd fprintf(stderr, "Unknown command: 0x%02X\n", cmd); @@ -348,13 +435,14 @@ void message_update(uint8_t* buffer, size_t len) { // recv 5 bytes data[data_itr++] = byte; if (data_itr >= 5) { - data_len = *(uint32_t*)(data + 1) + 5; + data_len = *(uint32_t*)(data + 1); if (data_len == 0) { // exec exec_cmd(); state = 0; break; } + data_len += 5; // check capacity if (data_len > data_cap) { data = realloc(data, data_len); @@ -425,81 +513,80 @@ void on_new_connection(uv_stream_t *server, int status) { } } -uint8_t pic_write_buffer[PIC_YUV_SIZE + 6] = { - 0x99, 0x9a, - PIC_YUV_SIZE & 0xff, - (PIC_YUV_SIZE >> 8) & 0xff, - (PIC_YUV_SIZE >> 16) & 0xff, - (PIC_YUV_SIZE >> 24) & 0xff -}; void stdin_read(uv_stream_t *stream, ssize_t nread, const uv_buf_t* buf) { if (nread < 0) { uv_close((uv_handle_t*)stream, NULL); return; } + static uint32_t ptr = 0; if (flag_send_pic == 0) { // do not send - return; + ptr = 0; + goto clean; } - uint8_t* pic_nv12_buffer = pic_write_buffer + 6; - static uint32_t ptr = 0; + uint8_t* pic_nv12_buffer = pic_write_buffer[pic_write_buffer_select] + 6; size_t lack = PIC_YUV_SIZE - ptr; -#if DEBUG_STDIN - // send any way - if (nread >= lack) { - memcpy(pic_nv12_buffer + ptr, buf->base, lack); - } else { - memcpy(pic_nv12_buffer + ptr, buf->base, nread); - } - // broadcast - broadcast(pic_write_buffer, sizeof(pic_write_buffer), 1); - static uint32_t pic_num = 0; - printf("\rpicture sent, #%d", pic_num++); - fflush(stdout); - ptr = 0; -#else if (nread >= lack) { static time_t last_time = 0; if (time(NULL) - last_time >= 1) { memcpy(pic_nv12_buffer + ptr, buf->base, lack); // broadcast - broadcast(pic_write_buffer, sizeof(pic_write_buffer), 1); + broadcast(pic_write_buffer[pic_write_buffer_select], sizeof(pic_write_buffer[0]), 1); static uint32_t pic_num = 0; - printf("\rpicture sent, #%d", pic_num++); + printf("\rreceived pictures, #%d", pic_num++); fflush(stdout); last_time = time(NULL); } + pic_write_buffer_select ^= 1; ptr = 0; } else { memcpy(pic_nv12_buffer + ptr, buf->base, nread); ptr += nread; } -#endif +clean: if (buf->base) { free(buf->base); } } void idle_mock_stdin (uv_idle_t* idle) { - uv_buf_t buf = { - .base = malloc(4), - .len = 4 - }; - stdin_read(NULL, 4, &buf); + char *stack_dirty = malloc(PIC_YUV_SIZE); + uv_buf_t buf = {.base = stack_dirty, .len = PIC_YUV_SIZE}; + stdin_read(NULL, PIC_YUV_SIZE, &buf); } +int share_mem_fd = -1, mem_fd = -1; + void before_exit(int sig) { + if (henc != NULL) { + VideoEncoder_Destroy(henc); + } uv_loop_close(loop); + if (map_base != MAP_FAILED) { + munmap(map_base, map_size); + close(mem_fd); + } + if (mem_yuv_logic_addr != MAP_FAILED) { + munmap(mem_yuv_logic_addr, mem_yuv_map_size); + } + if (mem_yuv_map_size != 0) { + ioctl(share_mem_fd, SHARE_MEMORY_FREE, mem_yuv_phy_addr); + close(share_mem_fd); + } fprintf(stderr, "capture signal %d, exit\n", sig); exit(sig); } -int main(int argc, char* argv[]) { +int main(int argc, char *argv[]) { + loop = uv_default_loop(); + signal(SIGINT, before_exit); + signal(SIGTERM, before_exit); + signal(SIGPIPE, SIG_IGN); #if QEMU puts("In emulator, will not try to set register"); #else - // map memory + // isp register memory page_size = getpagesize(); offset_in_page = (unsigned)REGISTER_BASE_ADDR & (page_size - 1); if (offset_in_page + 32 > page_size) { @@ -508,10 +595,43 @@ int main(int argc, char* argv[]) { printf("page size: %d\noff_t size: %ld\n" "offset_in_page: %d\nmap_size: %d\n", page_size, sizeof(off_t), offset_in_page, map_size); - int mem_fd = open("/dev/mem", O_RDWR | O_SYNC); + mem_fd = open("/dev/mem", O_RDWR | O_SYNC); assert(mem_fd >= 0); map_base = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, mem_fd, REGISTER_BASE_ADDR & ~(off_t)(page_size - 1)); assert(map_base != MAP_FAILED); + // for hd encoder + share_mem_fd = open("/dev/k510-share-memory", O_RDWR | O_SYNC); + assert(share_mem_fd >= 0); + assert(alloc_memory(share_mem_fd, PIC_YUV_SIZE, &mem_yuv_phy_addr, + &mem_yuv_map_size) == 0); + mem_yuv_logic_addr = mmap(NULL, mem_yuv_map_size, + PROT_READ | PROT_WRITE, MAP_SHARED, + mem_fd, (uint64_t)mem_yuv_phy_addr | 0x100000000); + assert(mem_yuv_logic_addr != MAP_FAILED); + + // open encoder + EncSettings encoder_settings = { + .channel = 0, // TODO + .width = PIC_YUV_WIDTH, + .height = PIC_YUV_HEIGHT, + .FrameRate = 30, + .BitRate = 4000000, // TODO + .MaxBitRate = 4000000, + .level = 42, + .profile = JPEG, + .rcMode = CONST_QP, + .SliceQP = 25, + .FreqIDR = 25, + .gopLen = 25, + .AspectRatio = ASPECT_RATIO_AUTO, + .MinQP = 0, + .MaxQP = 51, + .bEnableGDR = 0, + .gdrMode = GDR_VERTICAL, + .bEnableLTR = 0, + .roiCtrlMode = ROI_QP_TABLE_NONE, + }; + henc = VideoEncoder_Create(&encoder_settings); #endif // init @@ -520,10 +640,6 @@ int main(int argc, char* argv[]) { list_init(&clients); // socket - loop = uv_default_loop(); - signal(SIGINT, before_exit); - signal(SIGTERM, before_exit); - signal(SIGPIPE, SIG_IGN); uv_tcp_t server; uv_tcp_init(loop, &server); uv_ip4_addr("0.0.0.0", DEFAULT_PORT, &addr); @@ -544,7 +660,6 @@ int main(int argc, char* argv[]) { #if DEBUG_STDIN uv_idle_t idle_task; uv_idle_init(loop, &idle_task); - uv_idle_cb c; uv_idle_start(&idle_task, idle_mock_stdin); #endif return uv_run(loop, UV_RUN_DEFAULT); diff --git a/package/mediactl_lib/Config.in b/package/mediactl_lib/Config.in index 45614bc..db23c7a 100644 --- a/package/mediactl_lib/Config.in +++ b/package/mediactl_lib/Config.in @@ -2,6 +2,5 @@ config BR2_PACKAGE_MEDIACTL_LIB bool "mediactl_lib" depends on BR2_riscv - select BR2_PACKAGE_LIBUV help k510 mediactl lib for demo diff --git a/package/mediactl_lib/mediactl_lib.mk b/package/mediactl_lib/mediactl_lib.mk index cc238a6..0763036 100644 --- a/package/mediactl_lib/mediactl_lib.mk +++ b/package/mediactl_lib/mediactl_lib.mk @@ -34,7 +34,6 @@ define MEDIACTL_LIB_INSTALL_TARGET_CMDS $(INSTALL) -D -m 0755 $(@D)/v4l2_test/v4l2_test.out $(TARGET_DIR)/app/mediactl_lib/ $(INSTALL) -D -m 0755 $(@D)/v4l2_test/video_test.conf $(TARGET_DIR)/app/mediactl_lib/ $(INSTALL) -D -m 0755 $(@D)/v4l2_drm_isptool/v4l2_drm_isptool.out $(TARGET_DIR)/app/mediactl_lib/ - $(INSTALL) -D -m 0755 $(@D)/v4l2_drm_isptool/isp-tuningd $(TARGET_DIR)/app/mediactl_lib/ endef $(eval $(generic-package)) diff --git a/package/mediactl_lib/src/v4l2_drm_isptool/Makefile b/package/mediactl_lib/src/v4l2_drm_isptool/Makefile index 1401c79..3d2a1b5 100644 --- a/package/mediactl_lib/src/v4l2_drm_isptool/Makefile +++ b/package/mediactl_lib/src/v4l2_drm_isptool/Makefile @@ -6,13 +6,7 @@ OBJS = v4l2_drm_isptool.c ../v4l2_drm/drm/k510_drm.c ../v4l2_drm/sequeue.c all: ${CC} ${OBJS} -I../v4l2_drm/ -L. -L../ -I../v4l2_drm/drm -ldrm -lmediactl -lpthread -lstdc++ -o ${TARGET} - ${CC} isp-tuningd.c -lpthread -luv -o isp-tuningd @echo "Compile done." clean: @rm -f *.o ${TARGET} @echo "Clean done." - -v4l2_drm: - -isp-tuningd: -