Nvidia Jetson Nano gstreamer + TensorRT support

.github/workflows/pull_request.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Build and run tests
-        run: make run_tests PLATFORM="linux/arm64/v8" ARCH="aarch64"
+        run: make run_tests PLATFORM="linux/arm64/v8" ARCH="aarch64" FFMPEG_ARCH="arm64"
 
   docker_tests_on_amd64:
     runs-on: ubuntu-latest
@@ -67,4 +67,4 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Build and run tests
-        run: make run_tests PLATFORM="linux/amd64" ARCH="amd64"
+        run: make run_tests PLATFORM="linux/amd64" ARCH="amd64" FFMPEG_ARCH="amd64"

Makefile

@@ -47,6 +47,29 @@ aarch64_frigate: version web
 
 aarch64_all: aarch64_wheels aarch64_ffmpeg aarch64_frigate
 
+aarch64_l4t_assets:
+	mkdir -p $$(pwd)/.l4t_assets
+	cp ./converters/yolo4/plugin/* .l4t_assets/
+	cp ./converters/yolo4/model/yolov4-tiny-416.trt .l4t_assets/yolov4-tiny-416.trt
+	cp ./converters/yolo4/model/yolov4-tiny-288.trt .l4t_assets/yolov4-tiny-288.trt
+	# cp ./converters/yolo4/model/yolov4-416.trt .l4t_assets/yolov4-416.trt
+	# cp ./converters/yolo4/model/yolov4-288.trt .l4t_assets/yolov4-288.trt
+
+aarch64_l4t_wheels:
+	@docker build --tag frigate-wheels-l4t --file docker/Dockerfile.wheels.l4t .
+	# Run l4t wheels using nvidia runtime
+	@docker rm frigate.wheels.l4t || true
+	@docker run --name frigate.wheels.l4t -it --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all -e NVIDIA_DRIVER_CAPABILITIES=compute,utility,video --privileged frigate-wheels-l4t
+	# Commit changes to the container
+	@CONTAINER_ID=`docker ps -n 1 --format "{{.ID}}"`
+	@docker commit $$CONTAINER_ID frigate.wheels.l4t frigate-wheels-l4t:latest
+	@docker rm frigate.wheels.l4t || true
+
+aarch64_l4t_frigate: # aarch64_l4t_wheels aarch64_l4t_assets
+	docker build --tag frigate-base-l4t --build-arg BASE_IMAGE=timongentzsch/l4t-ubuntu20-opencv:latest --build-arg FFMPEG_ARCH=arm64 --build-arg ARCH=aarch64 --build-arg WHEELS_VERSION=1.0.3 --build-arg NGINX_VERSION=1.0.2 --file docker/Dockerfile.base .
+	nvidia-docker build --tag frigate.l4t --build-arg NGINX_VERSION=1.0.2 --file docker/Dockerfile.aarch64.l4t .
+
+
 armv7_wheels:
 	docker build --tag blakeblackshear/frigate-wheels:1.0.3-armv7 --file docker/Dockerfile.wheels .
 
@@ -68,7 +91,7 @@ run_tests:
 	@sed -i "s/FROM frigate-base/#/g" docker/Dockerfile.test
 	@echo "" >> docker/Dockerfile.test
 	@echo "RUN python3 -m unittest" >> docker/Dockerfile.test
-	@docker buildx build --platform=$(PLATFORM) --tag frigate-base --build-arg NGINX_VERSION=1.0.2 --build-arg FFMPEG_VERSION=1.0.0 --build-arg ARCH=$(ARCH) --build-arg WHEELS_VERSION=1.0.3 --file docker/Dockerfile.test .
+	@docker buildx build --platform=$(PLATFORM) --tag frigate-base --build-arg NGINX_VERSION=1.0.2 --build-arg FFMPEG_VERSION=1.0.0 --build-arg ARCH=$(ARCH) --build-arg FFMPEG_ARCH=$(FFMPEG_ARCH) --build-arg WHEELS_VERSION=1.0.3 --file docker/Dockerfile.test .
 	@rm docker/Dockerfile.test
 
 .PHONY: web run_tests

converters/ssd_mobilenet_v2_coco/Dockerfile.l4t.tf15

@@ -0,0 +1,10 @@
+FROM nvcr.io/nvidia/l4t-tensorflow:r32.6.1-tf1.15-py3
+
+RUN apt-get update && apt-get install -y git sudo
+RUN git clone https://github.com/jkjung-avt/tensorrt_demos.git /tensorrt_demos
+
+ADD 0001-fix-trt.patch /tensorrt_demos/0001-fix-trt.patch
+RUN cd /tensorrt_demos && \
+    git apply 0001-fix-trt.patch
+
+ADD run.sh /run.sh

converters/ssd_mobilenet_v2_coco/README.md

@@ -0,0 +1,14 @@
+
+A build.sh file will convert pre-trained tensorflow Single-Shot Multibox Detector (SSD) models through UFF to TensorRT engine to do real-time object detection with the TensorRT engine.
+
+Output will be copied to the ./model folder
+
+
+Note:
+
+This will consume pretty significant amound of memory. You might consider extending swap on Jetson Nano
+
+Usage:
+
+cd ./frigate/converters/ssd_mobilenet_v2_coco/
+./build.sh

converters/ssd_mobilenet_v2_coco/assets/0001-fix-trt.patch

@@ -0,0 +1,52 @@
+From 40953eaae8ca55838e046325b257faaff0bbe33f Mon Sep 17 00:00:00 2001
+From: YS <ys@gm.com>
+Date: Tue, 21 Dec 2021 21:01:35 +0300
+Subject: [PATCH] fix trt
+
+---
+ ssd/build_engine.py | 13 ++++++++-----
+ 1 file changed, 8 insertions(+), 5 deletions(-)
+
+diff --git a/ssd/build_engine.py b/ssd/build_engine.py
+index 65729a9..e4a55c8 100644
+--- a/ssd/build_engine.py
++++ b/ssd/build_engine.py
+@@ -17,7 +17,6 @@ import uff
+ import tensorrt as trt
+ import graphsurgeon as gs
+
+-
+ DIR_NAME = os.path.dirname(__file__)
+ LIB_FILE = os.path.abspath(os.path.join(DIR_NAME, 'libflattenconcat.so'))
+ MODEL_SPECS = {
+@@ -286,19 +285,23 @@ def main():
+         text=True,
+         debug_mode=DEBUG_UFF)
+     with trt.Builder(TRT_LOGGER) as builder, builder.create_network() as network, trt.UffParser() as parser:
+-        builder.max_workspace_size = 1 << 28
++        config = builder.create_builder_config()
++        config.max_workspace_size = 1 << 28
+         builder.max_batch_size = 1
+-        builder.fp16_mode = True
++        config.set_flag(trt.BuilderFlag.FP16)
+
+         parser.register_input('Input', INPUT_DIMS)
+         parser.register_output('MarkOutput_0')
+         parser.parse(spec['tmp_uff'], network)
+-        engine = builder.build_cuda_engine(network)
++
++        plan = builder.build_serialized_network(network, config)
++
++        with trt.Runtime(TRT_LOGGER) as runtime:
++            engine = runtime.deserialize_cuda_engine(plan)
+
+         buf = engine.serialize()
+         with open(spec['output_bin'], 'wb') as f:
+             f.write(buf)
+
+-
+ if __name__ == '__main__':
+     main()
+-- 
+2.17.1
+

converters/ssd_mobilenet_v2_coco/assets/run.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -xe
+cd /tensorrt_demos/ssd
+./install.sh
+python3 build_engine.py ssd_mobilenet_v2_coco
+cp /tensorrt_demos/ssd/TRT_ssd_mobilenet_v2_coco.bin /model/TRT_ssd_mobilenet_v2_coco.bin

converters/ssd_mobilenet_v2_coco/build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+mkdir -p $(pwd)/model
+
+docker build --tag models.ssd_v2_coco  --file ./Dockerfile.l4t.tf15 ./assets/
+
+sudo docker run --rm -it --name models.ssd_v2_coco \
+    --mount type=tmpfs,target=/tmp/cache,tmpfs-size=1000000000 \
+    -v $(pwd)/model:/model:rw \
+    -v /tmp/argus_socket:/tmp/argus_socket \
+    -e NVIDIA_VISIBLE_DEVICES=all \
+    -e NVIDIA_DRIVER_CAPABILITIES=compute,utility,video \
+    --runtime=nvidia \
+    --privileged \
+    models.ssd_v2_coco /run.sh

converters/yolo4/Dockerfile.l4t.tf15

@@ -0,0 +1,14 @@
+FROM nvcr.io/nvidia/l4t-tensorflow:r32.6.1-tf1.15-py3
+
+RUN apt-get update && apt-get install -y git sudo
+RUN git clone https://github.com/jkjung-avt/tensorrt_demos.git /tensorrt_demos
+
+RUN cd /tensorrt_demos/yolo && ./install_pycuda.sh
+RUN apt-get update && apt-get install -y cmake build-essential unzip
+ADD install_protobuf.sh /install_protobuf.sh
+RUN /install_protobuf.sh
+RUN pip3 install onnx==1.4.1
+RUN cd /tensorrt_demos/yolo && ./download_yolo.sh
+ADD run.sh /run.sh
+
+

converters/yolo4/README.md

@@ -0,0 +1,45 @@
+Following the https://github.com/jkjung-avt/tensorrt_demos#demo-5-yolov4
+
+
+A build.sh file will convert pre-trained yolov3 and yolov4 models through ONNX to TensorRT engines. The implementation with a "yolo_layer" plugin has been updated to speed up inference time of the yolov3/yolov4 models.
+
+Current "yolo_layer" plugin implementation is based on TensorRT's IPluginV2IOExt. It only works for TensorRT 6+. "yolo_layer" developed plugin by referencing similar plugin code by wang-xinyu and dongfangduoshou123. So big thanks to both of them.
+
+
+Output will be copied to the ./model folder
+
+
+
+## Available models
+
+   | TensorRT engine         | mAP @<br>IoU=0.5:0.95 |  mAP @<br>IoU=0.5  | FPS on Nano |
+   |:------------------------|:---------------------:|:------------------:|:-----------:|
+   | yolov3-tiny-288 (FP16)  |         0.077         |        0.158       |     35.8    |
+   | yolov3-tiny-416 (FP16)  |         0.096         |        0.202       |     25.5    |
+   | yolov3-288 (FP16)       |         0.331         |        0.601       |     8.16    |
+   | yolov3-416 (FP16)       |         0.373         |        0.664       |     4.93    |
+   | yolov3-608 (FP16)       |         0.376         |        0.665       |     2.53    |
+   | yolov3-spp-288 (FP16)   |         0.339         |        0.594       |     8.16    |
+   | yolov3-spp-416 (FP16)   |         0.391         |        0.664       |     4.82    |
+   | yolov3-spp-608 (FP16)   |         0.410         |        0.685       |     2.49    |
+   | yolov4-tiny-288 (FP16)  |         0.179         |        0.344       |     36.6    |
+   | yolov4-tiny-416 (FP16)  |         0.196         |        0.387       |     25.5    |
+   | yolov4-288 (FP16)       |         0.376         |        0.591       |     7.93    |
+   | yolov4-416 (FP16)       |         0.459         |        0.700       |     4.62    |
+   | yolov4-608 (FP16)       |         0.488         |        0.736       |     2.35    |
+   | yolov4-csp-256 (FP16)   |         0.336         |        0.502       |     12.8    |
+   | yolov4-csp-512 (FP16)   |         0.436         |        0.630       |     4.26    |
+   | yolov4x-mish-320 (FP16) |         0.400         |        0.581       |     4.79    |
+   | yolov4x-mish-640 (FP16) |         0.470         |        0.668       |     1.46    |
+
+
+Please update frigate/converters/yolo4/assets/run.sh to add necessary models
+
+Note:
+
+This will consume pretty significant amound of memory. You might consider extending swap on Jetson Nano
+
+Usage:
+
+cd ./frigate/converters/yolo4/
+./build.sh

converters/yolo4/assets/install_protobuf.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -e
+
+folder=${HOME}/src
+mkdir -p $folder
+
+echo "** Install requirements"
+sudo apt-get install -y autoconf libtool
+
+echo "** Download protobuf-3.8.0 sources"
+cd $folder
+if [ ! -f protobuf-python-3.8.0.zip ]; then
+  wget https://github.com/protocolbuffers/protobuf/releases/download/v3.8.0/protobuf-python-3.8.0.zip
+fi
+if [ ! -f protoc-3.8.0-linux-aarch_64.zip ]; then
+  wget https://github.com/protocolbuffers/protobuf/releases/download/v3.8.0/protoc-3.8.0-linux-aarch_64.zip
+fi
+
+echo "** Install protoc"
+unzip protobuf-python-3.8.0.zip
+unzip protoc-3.8.0-linux-aarch_64.zip -d protoc-3.8.0
+sudo cp protoc-3.8.0/bin/protoc /usr/local/bin/protoc
+
+echo "** Build and install protobuf-3.8.0 libraries"
+export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+cd protobuf-3.8.0/
+./autogen.sh
+./configure --prefix=/usr/local
+make -j$(nproc)
+make check
+sudo make install
+sudo ldconfig
+
+echo "** Update python3 protobuf module"
+# remove previous installation of python3 protobuf module
+sudo apt-get install -y python3-pip
+sudo pip3 uninstall -y protobuf
+sudo pip3 install Cython
+cd python/
+python3 setup.py build --cpp_implementation
+python3 setup.py test --cpp_implementation
+sudo python3 setup.py install --cpp_implementation
+
+echo "** Build protobuf-3.8.0 successfully"

converters/yolo4/assets/run.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -xe
+cd /tensorrt_demos/plugins && make
+cp /tensorrt_demos/plugins/libyolo_layer.so /plugin/libyolo_layer.so
+
+cd /tensorrt_demos/yolo
+for model in yolov4-tiny-288 \
+             yolov4-tiny-416 \
+             yolov4-288 \
+             yolov4-416 ; do
+    python3 yolo_to_onnx.py -m ${model}
+    python3 onnx_to_tensorrt.py -m ${model}
+    cp /tensorrt_demos/yolo/${model}.trt /model/${model}.trt
+done     

converters/yolo4/build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+mkdir -p $(pwd)/model
+mkdir -p $(pwd)/plugin
+
+docker build --tag models.yolo4  --file ./Dockerfile.l4t.tf15 ./assets/
+
+sudo docker run --rm -it --name models.yolo4 \
+    --mount type=tmpfs,target=/tmp/cache,tmpfs-size=1000000000 \
+    -v $(pwd)/model:/model:rw \
+    -v $(pwd)/plugin:/plugin:rw \
+    -v /tmp/argus_socket:/tmp/argus_socket \
+    -e NVIDIA_VISIBLE_DEVICES=all \
+    -e NVIDIA_DRIVER_CAPABILITIES=compute,utility,video \
+    --runtime=nvidia \
+    --privileged \
+    models.yolo4 /run.sh

docker/Dockerfile.aarch64.l4t

@@ -0,0 +1,43 @@
+FROM frigate-wheels-l4t:latest as wheelsl4t
+FROM frigate-base-l4t
+LABEL maintainer "blakeb@blakeshome.com"
+
+COPY --from=wheelsl4t /wheels/. /wheels/
+
+ENV DEBIAN_FRONTEND=noninteractive
+# Install packages for apt repo
+RUN apt-get -qq update \
+    && apt-get -qq install --no-install-recommends -y \
+    git \
+    # ffmpeg runtime dependencies
+    libgomp1 \
+    # runtime dependencies
+    libopenexr24 \
+    libgstreamer1.0-0 \
+    libgstreamer-plugins-base1.0-0 \
+    libopenblas-base \
+    libjpeg-turbo8 \
+    libpng16-16 \
+    libtiff5 \
+    libdc1394-22 \
+    && rm -rf /var/lib/apt/lists/* \
+    && (apt-get autoremove -y; apt-get autoclean -y)
+
+# Install tensorrt and pycuda
+RUN pip3 install -U /wheels/*.whl
+
+# Yolo4 model and plugins
+ADD .l4t_assets /yolo4/
+
+# s6-overlay
+ADD https://github.com/just-containers/s6-overlay/releases/download/v2.2.0.3/s6-overlay-aarch64-installer /tmp/
+RUN chmod +x /tmp/s6-overlay-aarch64-installer && /tmp/s6-overlay-aarch64-installer /
+
+# Set up library path for the GStreamer
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/aarch64-linux-gnu/tegra
+
+ENTRYPOINT ["/init"]
+
+CMD ["python3", "-u", "-m", "frigate"]
+
+COPY frigate frigate/