diff --git a/Makefile b/Makefile index 3c3f8c04ae..7f7baafc24 100644 --- a/Makefile +++ b/Makefile @@ -8,22 +8,32 @@ version: nginx_frigate: docker buildx build --push --platform linux/arm/v7,linux/arm64/v8,linux/amd64 --tag blakeblackshear/frigate-nginx:1.0.2 --file docker/Dockerfile.nginx . +l4t_assets: + mkdir -p $$(pwd)/.l4t_assets + cp ./converters/yolo4/plugin/* .l4t_assets/ + cp ./converters/yolo4/model/yolov4-tiny-416.trt .l4t_assets/yolov4-tiny-416.trt + cp ./converters/yolo4/model/yolov4-tiny-288.trt .l4t_assets/yolov4-tiny-288.trt + # cp ./converters/yolo4/model/yolov4-416.trt .l4t_assets/yolov4-416.trt + # cp ./converters/yolo4/model/yolov4-288.trt .l4t_assets/yolov4-288.trt + +l4t_wheels: + @docker build --tag frigate-wheels-l4t --file docker/Dockerfile.wheels.l4t . + # Run l4t wheels using nvidia runtime + @docker rm frigate.wheels.l4t || true + @docker run --name frigate.wheels.l4t -it --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all -e NVIDIA_DRIVER_CAPABILITIES=compute,utility,video --privileged frigate-wheels-l4t + # Commit changes to the container + @CONTAINER_ID=`docker ps -n 1 --format "{{.ID}}"` + @docker commit $$CONTAINER_ID frigate.wheels.l4t frigate-wheels-l4t:latest + @docker rm frigate.wheels.l4t || true + +l4t_frigate: l4t_wheels l4t_assets + @cat docker/Dockerfile | sed "s|#use_l4t: ||g" > docker/Dockerfile.l4t + DOCKER_BUILDKIT=1 docker build --progress=plain -t frigate.l4t --build-arg FRIGATE_BASE_IMAGE=timongentzsch/l4t-ubuntu20-opencv:latest -f docker/Dockerfile.l4t . + frigate: version - DOCKER_BUILDKIT=1 docker build -t frigate -f docker/Dockerfile . + DOCKER_BUILDKIT=1 docker build --progress=plain -t frigate -f docker/Dockerfile . frigate_push: version docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag blakeblackshear/frigate:0.11.0-$(COMMIT_HASH) --file docker/Dockerfile . -run_tests: - # PLATFORM: linux/arm64/v8 linux/amd64 or linux/arm/v7 - # ARCH: aarch64 amd64 or armv7 - @cat docker/Dockerfile.base docker/Dockerfile.$(ARCH) > docker/Dockerfile.test - @sed -i "s/FROM frigate-web as web/#/g" docker/Dockerfile.test - @sed -i "s/COPY --from=web \/opt\/frigate\/build web\//#/g" docker/Dockerfile.test - @sed -i "s/FROM frigate-base/#/g" docker/Dockerfile.test - @echo "" >> docker/Dockerfile.test - @echo "RUN python3 -m unittest" >> docker/Dockerfile.test - @docker buildx build --platform=$(PLATFORM) --tag frigate-base --build-arg NGINX_VERSION=1.0.2 --build-arg FFMPEG_VERSION=1.0.0 --build-arg ARCH=$(ARCH) --build-arg WHEELS_VERSION=1.0.3 --file docker/Dockerfile.test . - @rm docker/Dockerfile.test - -.PHONY: run_tests +.PHONY: run_tests l4t_frigate diff --git a/converters/ssd_mobilenet_v2_coco/Dockerfile.l4t.tf15 b/converters/ssd_mobilenet_v2_coco/Dockerfile.l4t.tf15 new file mode 100644 index 0000000000..0eac9ce6d2 --- /dev/null +++ b/converters/ssd_mobilenet_v2_coco/Dockerfile.l4t.tf15 @@ -0,0 +1,10 @@ +FROM nvcr.io/nvidia/l4t-tensorflow:r32.6.1-tf1.15-py3 + +RUN apt-get update && apt-get install -y git sudo +RUN git clone https://github.com/jkjung-avt/tensorrt_demos.git /tensorrt_demos + +ADD 0001-fix-trt.patch /tensorrt_demos/0001-fix-trt.patch +RUN cd /tensorrt_demos && \ + git apply 0001-fix-trt.patch + +ADD run.sh /run.sh diff --git a/converters/ssd_mobilenet_v2_coco/README.md b/converters/ssd_mobilenet_v2_coco/README.md new file mode 100644 index 0000000000..188b17b4e4 --- /dev/null +++ b/converters/ssd_mobilenet_v2_coco/README.md @@ -0,0 +1,14 @@ + +A build.sh file will convert pre-trained tensorflow Single-Shot Multibox Detector (SSD) models through UFF to TensorRT engine to do real-time object detection with the TensorRT engine. + +Output will be copied to the ./model folder + + +Note: + +This will consume pretty significant amound of memory. You might consider extending swap on Jetson Nano + +Usage: + +cd ./frigate/converters/ssd_mobilenet_v2_coco/ +./build.sh \ No newline at end of file diff --git a/converters/ssd_mobilenet_v2_coco/assets/0001-fix-trt.patch b/converters/ssd_mobilenet_v2_coco/assets/0001-fix-trt.patch new file mode 100644 index 0000000000..a436cb01bd --- /dev/null +++ b/converters/ssd_mobilenet_v2_coco/assets/0001-fix-trt.patch @@ -0,0 +1,52 @@ +From 40953eaae8ca55838e046325b257faaff0bbe33f Mon Sep 17 00:00:00 2001 +From: YS +Date: Tue, 21 Dec 2021 21:01:35 +0300 +Subject: [PATCH] fix trt + +--- + ssd/build_engine.py | 13 ++++++++----- + 1 file changed, 8 insertions(+), 5 deletions(-) + +diff --git a/ssd/build_engine.py b/ssd/build_engine.py +index 65729a9..e4a55c8 100644 +--- a/ssd/build_engine.py ++++ b/ssd/build_engine.py +@@ -17,7 +17,6 @@ import uff + import tensorrt as trt + import graphsurgeon as gs + +- + DIR_NAME = os.path.dirname(__file__) + LIB_FILE = os.path.abspath(os.path.join(DIR_NAME, 'libflattenconcat.so')) + MODEL_SPECS = { +@@ -286,19 +285,23 @@ def main(): + text=True, + debug_mode=DEBUG_UFF) + with trt.Builder(TRT_LOGGER) as builder, builder.create_network() as network, trt.UffParser() as parser: +- builder.max_workspace_size = 1 << 28 ++ config = builder.create_builder_config() ++ config.max_workspace_size = 1 << 28 + builder.max_batch_size = 1 +- builder.fp16_mode = True ++ config.set_flag(trt.BuilderFlag.FP16) + + parser.register_input('Input', INPUT_DIMS) + parser.register_output('MarkOutput_0') + parser.parse(spec['tmp_uff'], network) +- engine = builder.build_cuda_engine(network) ++ ++ plan = builder.build_serialized_network(network, config) ++ ++ with trt.Runtime(TRT_LOGGER) as runtime: ++ engine = runtime.deserialize_cuda_engine(plan) + + buf = engine.serialize() + with open(spec['output_bin'], 'wb') as f: + f.write(buf) + +- + if __name__ == '__main__': + main() +-- +2.17.1 + diff --git a/converters/ssd_mobilenet_v2_coco/assets/run.sh b/converters/ssd_mobilenet_v2_coco/assets/run.sh new file mode 100755 index 0000000000..28f09ad77f --- /dev/null +++ b/converters/ssd_mobilenet_v2_coco/assets/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -xe +cd /tensorrt_demos/ssd +./install.sh +python3 build_engine.py ssd_mobilenet_v2_coco +cp /tensorrt_demos/ssd/TRT_ssd_mobilenet_v2_coco.bin /model/TRT_ssd_mobilenet_v2_coco.bin \ No newline at end of file diff --git a/converters/ssd_mobilenet_v2_coco/build.sh b/converters/ssd_mobilenet_v2_coco/build.sh new file mode 100755 index 0000000000..8f259c5a14 --- /dev/null +++ b/converters/ssd_mobilenet_v2_coco/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +mkdir -p $(pwd)/model + +docker build --tag models.ssd_v2_coco --file ./Dockerfile.l4t.tf15 ./assets/ + +sudo docker run --rm -it --name models.ssd_v2_coco \ + --mount type=tmpfs,target=/tmp/cache,tmpfs-size=1000000000 \ + -v $(pwd)/model:/model:rw \ + -v /tmp/argus_socket:/tmp/argus_socket \ + -e NVIDIA_VISIBLE_DEVICES=all \ + -e NVIDIA_DRIVER_CAPABILITIES=compute,utility,video \ + --runtime=nvidia \ + --privileged \ + models.ssd_v2_coco /run.sh diff --git a/converters/yolo4/Dockerfile.l4t.tf15 b/converters/yolo4/Dockerfile.l4t.tf15 new file mode 100644 index 0000000000..a4ced0f021 --- /dev/null +++ b/converters/yolo4/Dockerfile.l4t.tf15 @@ -0,0 +1,14 @@ +FROM nvcr.io/nvidia/l4t-tensorflow:r32.6.1-tf1.15-py3 + +RUN apt-get update && apt-get install -y git sudo +RUN git clone https://github.com/jkjung-avt/tensorrt_demos.git /tensorrt_demos + +RUN cd /tensorrt_demos/yolo && ./install_pycuda.sh +RUN apt-get update && apt-get install -y cmake build-essential unzip +ADD install_protobuf.sh /install_protobuf.sh +RUN /install_protobuf.sh +RUN pip3 install onnx==1.4.1 +RUN cd /tensorrt_demos/yolo && ./download_yolo.sh +ADD run.sh /run.sh + + diff --git a/converters/yolo4/README.md b/converters/yolo4/README.md new file mode 100644 index 0000000000..d3dc6cc666 --- /dev/null +++ b/converters/yolo4/README.md @@ -0,0 +1,45 @@ +Following the https://github.com/jkjung-avt/tensorrt_demos#demo-5-yolov4 + + +A build.sh file will convert pre-trained yolov3 and yolov4 models through ONNX to TensorRT engines. The implementation with a "yolo_layer" plugin has been updated to speed up inference time of the yolov3/yolov4 models. + +Current "yolo_layer" plugin implementation is based on TensorRT's IPluginV2IOExt. It only works for TensorRT 6+. "yolo_layer" developed plugin by referencing similar plugin code by wang-xinyu and dongfangduoshou123. So big thanks to both of them. + + +Output will be copied to the ./model folder + + + +## Available models + + | TensorRT engine | mAP @
IoU=0.5:0.95 | mAP @
IoU=0.5 | FPS on Nano | + |:------------------------|:---------------------:|:------------------:|:-----------:| + | yolov3-tiny-288 (FP16) | 0.077 | 0.158 | 35.8 | + | yolov3-tiny-416 (FP16) | 0.096 | 0.202 | 25.5 | + | yolov3-288 (FP16) | 0.331 | 0.601 | 8.16 | + | yolov3-416 (FP16) | 0.373 | 0.664 | 4.93 | + | yolov3-608 (FP16) | 0.376 | 0.665 | 2.53 | + | yolov3-spp-288 (FP16) | 0.339 | 0.594 | 8.16 | + | yolov3-spp-416 (FP16) | 0.391 | 0.664 | 4.82 | + | yolov3-spp-608 (FP16) | 0.410 | 0.685 | 2.49 | + | yolov4-tiny-288 (FP16) | 0.179 | 0.344 | 36.6 | + | yolov4-tiny-416 (FP16) | 0.196 | 0.387 | 25.5 | + | yolov4-288 (FP16) | 0.376 | 0.591 | 7.93 | + | yolov4-416 (FP16) | 0.459 | 0.700 | 4.62 | + | yolov4-608 (FP16) | 0.488 | 0.736 | 2.35 | + | yolov4-csp-256 (FP16) | 0.336 | 0.502 | 12.8 | + | yolov4-csp-512 (FP16) | 0.436 | 0.630 | 4.26 | + | yolov4x-mish-320 (FP16) | 0.400 | 0.581 | 4.79 | + | yolov4x-mish-640 (FP16) | 0.470 | 0.668 | 1.46 | + + +Please update frigate/converters/yolo4/assets/run.sh to add necessary models + +Note: + +This will consume pretty significant amound of memory. You might consider extending swap on Jetson Nano + +Usage: + +cd ./frigate/converters/yolo4/ +./build.sh \ No newline at end of file diff --git a/converters/yolo4/assets/install_protobuf.sh b/converters/yolo4/assets/install_protobuf.sh new file mode 100755 index 0000000000..245cabf4c1 --- /dev/null +++ b/converters/yolo4/assets/install_protobuf.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +folder=${HOME}/src +mkdir -p $folder + +echo "** Install requirements" +sudo apt-get install -y autoconf libtool + +echo "** Download protobuf-3.8.0 sources" +cd $folder +if [ ! -f protobuf-python-3.8.0.zip ]; then + wget https://github.com/protocolbuffers/protobuf/releases/download/v3.8.0/protobuf-python-3.8.0.zip +fi +if [ ! -f protoc-3.8.0-linux-aarch_64.zip ]; then + wget https://github.com/protocolbuffers/protobuf/releases/download/v3.8.0/protoc-3.8.0-linux-aarch_64.zip +fi + +echo "** Install protoc" +unzip protobuf-python-3.8.0.zip +unzip protoc-3.8.0-linux-aarch_64.zip -d protoc-3.8.0 +sudo cp protoc-3.8.0/bin/protoc /usr/local/bin/protoc + +echo "** Build and install protobuf-3.8.0 libraries" +export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp +cd protobuf-3.8.0/ +./autogen.sh +./configure --prefix=/usr/local +make -j$(nproc) +make check +sudo make install +sudo ldconfig + +echo "** Update python3 protobuf module" +# remove previous installation of python3 protobuf module +sudo apt-get install -y python3-pip +sudo pip3 uninstall -y protobuf +sudo pip3 install Cython +cd python/ +python3 setup.py build --cpp_implementation +python3 setup.py test --cpp_implementation +sudo python3 setup.py install --cpp_implementation + +echo "** Build protobuf-3.8.0 successfully" \ No newline at end of file diff --git a/converters/yolo4/assets/run.sh b/converters/yolo4/assets/run.sh new file mode 100755 index 0000000000..772ee43c02 --- /dev/null +++ b/converters/yolo4/assets/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -xe +cd /tensorrt_demos/plugins && make +cp /tensorrt_demos/plugins/libyolo_layer.so /plugin/libyolo_layer.so + +cd /tensorrt_demos/yolo +for model in yolov4-tiny-288 \ + yolov4-tiny-416 \ + yolov4-288 \ + yolov4-416 ; do + python3 yolo_to_onnx.py -m ${model} + python3 onnx_to_tensorrt.py -m ${model} + cp /tensorrt_demos/yolo/${model}.trt /model/${model}.trt +done diff --git a/converters/yolo4/build.sh b/converters/yolo4/build.sh new file mode 100755 index 0000000000..60d6887aa5 --- /dev/null +++ b/converters/yolo4/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +mkdir -p $(pwd)/model +mkdir -p $(pwd)/plugin + +docker build --tag models.yolo4 --file ./Dockerfile.l4t.tf15 ./assets/ + +sudo docker run --rm -it --name models.yolo4 \ + --mount type=tmpfs,target=/tmp/cache,tmpfs-size=1000000000 \ + -v $(pwd)/model:/model:rw \ + -v $(pwd)/plugin:/plugin:rw \ + -v /tmp/argus_socket:/tmp/argus_socket \ + -e NVIDIA_VISIBLE_DEVICES=all \ + -e NVIDIA_DRIVER_CAPABILITIES=compute,utility,video \ + --runtime=nvidia \ + --privileged \ + models.yolo4 /run.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index d758a8b5af..d454855fd5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,6 @@ +ARG FRIGATE_BASE_IMAGE=debian:11-slim +ARG L4T_WHEELS=frigate-wheels-l4t:latest +#use_l4t: FROM ${L4T_WHEELS} as wheels_l4t FROM blakeblackshear/frigate-nginx:1.0.2 as nginx FROM node:16 as web @@ -21,8 +24,7 @@ RUN apt-get -qq update \ && wget -O - http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - \ && echo "deb http://archive.raspberrypi.org/debian/ bullseye main" | tee /etc/apt/sources.list.d/raspi.list \ && apt-get -qq update \ - && apt-get -qq install -y \ - python3 \ + && apt-get -qq install -y python3 \ python3-dev \ wget \ # opencv dependencies @@ -60,12 +62,26 @@ RUN pip3 wheel --wheel-dir=/wheels \ ws4py # Frigate Container -FROM debian:11-slim +FROM ${FRIGATE_BASE_IMAGE} ARG TARGETARCH ENV DEBIAN_FRONTEND=noninteractive ENV FLASK_ENV=development +RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y ffmpeg + +# install gstreamer +#use_l4t: RUN \ +#use_l4t: apt-get update && apt-get install -y gstreamer1.0-plugins-base-apps gstreamer1.0-tools gstreamer1.0-alsa \ +#use_l4t: gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ +#use_l4t: gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly \ +#use_l4t: ibgstreamer1.0-dev \ +#use_l4t: libgstreamer-plugins-base1.0-dev \ +#use_l4t: libgstreamer-plugins-good1.0-dev \ +#use_l4t: libgstreamer-plugins-bad1.0-dev + +#use_l4t: RUN apt-get -qq update && apt-get -qq install -y python3.9 && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 +#use_l4t: COPY --from=wheels_l4t /wheels/. /wheels/ COPY --from=wheels /wheels /wheels # Install ffmpeg @@ -85,7 +101,6 @@ RUN apt-get -qq update \ && echo "libedgetpu1-max libedgetpu/accepted-eula select true" | debconf-set-selections \ && apt-get -qq update \ && apt-get -qq install --no-install-recommends -y \ - ffmpeg \ # coral drivers libedgetpu1-max python3-tflite-runtime python3-pycoral \ && pip3 install -U /wheels/*.whl \ @@ -104,7 +119,7 @@ RUN if [ "${TARGETARCH}" = "amd64" ]; \ && rm -rf /var/lib/apt/lists/* \ && (apt-get autoremove -y; apt-get autoclean -y) \ fi - + COPY --from=nginx /usr/local/nginx/ /usr/local/nginx/ # get model and labels @@ -120,6 +135,9 @@ COPY --from=web /opt/frigate/dist web/ COPY docker/rootfs/ / +#use_l4t: ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/aarch64-linux-gnu/tegra:/usr/lib/aarch64-linux-gnu +#use_l4t: ADD .l4t_assets /yolo4/ + # s6-overlay RUN S6_ARCH="${TARGETARCH}" \ && if [ "${TARGETARCH}" = "amd64" ]; then S6_ARCH="amd64"; fi \ diff --git a/docker/Dockerfile.wheels.l4t b/docker/Dockerfile.wheels.l4t new file mode 100644 index 0000000000..bec7b0531a --- /dev/null +++ b/docker/Dockerfile.wheels.l4t @@ -0,0 +1,36 @@ +FROM timongentzsch/l4t-ubuntu20-base:latest + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -yqq git python3.9 python3.9-dev python3-pip build-essential curl wget cmake +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 + +RUN \ + EXT_PATH=~/external && \ + mkdir -p $EXT_PATH && cd $EXT_PATH && \ + git clone https://github.com/pybind/pybind11.git + +RUN \ + PYTHON_VERSION=`python -c 'import platform;print(platform.python_version())'` && \ + EXT_PATH=~/external && \ + mkdir Python-${PYTHON_VERSION} && wget -qO- https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz | tar xvz --strip-components=1 -C Python-${PYTHON_VERSION} && \ + mkdir -p ${EXT_PATH}/python3.9/include && mv Python-${PYTHON_VERSION}/Include/* ${EXT_PATH}/python3.9/include/ && rm -rf Python-${PYTHON_VERSION} && \ + cp /usr/include/aarch64-linux-gnu/python3.9/pyconfig.h ~/external/python3.9/include/ + +RUN \ + mkdir /workspace && cd /workspace && git clone https://github.com/NVIDIA/TensorRT.git && \ + mkdir -p /workspace/TensorRT/python/include/onnx + +WORKDIR /workspace/TensorRT/python +# monkeypatch the environment +RUN \ + cd /workspace/TensorRT/python && \ + echo "update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1\n"\ + "update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1\n"\ + "mkdir -p /wheels && pip3 wheel --wheel-dir=/wheels pycuda==2021.1\n"\ + "cp -r /usr/local/cuda-10.2/targets/aarch64-linux/include/* ./include/\n"\ + "cp -r /usr/include/aarch64-linux-gnu/* ./include/onnx/\n"\ + "EXT_PATH=/root/external PYTHON_MAJOR_VERSION=3 PYTHON_MINOR_VERSION=9 TARGET_ARCHITECTURE=aarch64 ./build.sh\n" \ + "cp /workspace/TensorRT/python/build/dist/*.whl /wheels\n" >> ./bootstrap.sh && \ + chmod +x ./bootstrap.sh + +CMD ["/bin/bash", "./bootstrap.sh"] diff --git a/docs/docs/configuration/detectors.md b/docs/docs/configuration/detectors.md index 43a09d33cf..2d8e204555 100644 --- a/docs/docs/configuration/detectors.md +++ b/docs/docs/configuration/detectors.md @@ -64,6 +64,20 @@ detectors: device: pci ``` +### TensorRT on NVIDIA Jetson family devices + +```yaml +detectors: + JetsonNano: + type: tensorrt +``` + +You have to run TensorRT with the latest Jetpack version. + +**Note:** You might also want to set up the [GStreamer](/configuration/gstreamer) to get hardware-accelerated decoding on your Jetson device. + +_warning: TensorRT uses `trt` models. Please make sure you're using either the default `model` configuration or use proper `yolo4` models._ + ### CPU Detectors (not recommended) ```yaml diff --git a/docs/docs/configuration/gstreamer.md b/docs/docs/configuration/gstreamer.md new file mode 100644 index 0000000000..da2ac3c854 --- /dev/null +++ b/docs/docs/configuration/gstreamer.md @@ -0,0 +1,324 @@ +--- +id: gstreamer +title: GStreamer configuration +--- + +In addition to the FFmpeg, Frigate does support GStreamer. GStreamer is a framework for creating streaming media applications. The main advantages are that the pluggable components can be mixed and matched into arbitrary pipelines. +GStreamer has better support of hardware-accelerated video decoding on NVidia Jetson devices. + +**Note:** There are no advantages of GStreamer versus FFmpeg on non-Jetson devices. + +## Minimal GStreamer Configuration + +```yaml +camera_name: + gstreamer: + inputs: + - path: rtsp://10.0.0.1:554/stream0 + roles: + - detect + detect: + width: 1920 + height: 1080 +``` + +With the minimal configuration GStreamer integration will do the following: +- Run `gst-inspect-1.0` to get the available plugins +- Run `gst-discoverer-1.0` against the RTSP stream. The discovery process gets the audio and video stream codecs +- Build the GStreamer pipeline based on the available plugins. +- GStreamer automatically enable audio stream for recording if audio is available + +The default configuration should be enough for most of the cases. However, if you have multiple cameras, running `gst-discoverer-1.0` for each one might be pretty time-consuming. To avoid running `gst-discoverer-1.0`, you have to specify the video codec as well as the audio codec for the `record` role. + + +## GStreamer configuration with video and audio codecs set + +```yaml +camera_name: + gstreamer: + video_format: video/x-h265 + audio_format: audio/x-alaw + inputs: + - path: rtsp://10.0.0.1:554/stream0 + # video_format: video/x-h265 + # audio_format: audio/x-alaw + roles: + - record + + - path: rtsp://10.0.0.1:554/stream1 + # video_format: video/x-h265 + roles: + - detect + detect: + width: 1920 + height: 1080 +``` + +This setup is much more reliable, as it won't trigger the call to the `gst-discoverer-1.0` which might timeout occasionally. If you have different audio or video formats for different streams, you can override the defaults for each input. + +### Supported Video Formats + +GStreamer integration does not limit you which video format to use. It solely depends on the GStreamer plugins. GStreamer integration is building the `parse` plugin by doing the following steps: + +- lowercase the video format +- strip the optional `video/x-` prefix +- add the remainder to the `parse` for the `recorder` +- create"rtp{video_format}depay" element for the `decoder` + +That way, if you specified `video/x-h264` as a video format, GStreamer should support the `h264parse` and `rtph264depay` pipeline elements. + +### Supported Audio Formats + +As of now, only `audio/x-alaw` and `audio/mpeg` are supported. Audio formats require different pipelines. +To add a new audio format, one has to update `AUDIO_PIPELINES` in the `gstreamer.py`. + +Alternatively, `audio_pipeline` element of either input or camera level can be added to specify a custom audio pipeline. +`audio_pipeline` has a priority over `audio_format`, e.g. if you set both `audio_format` and `audio_pipeline`, the `audio_pipeline` will be used for decoding audio. + +Audio settings make sense only for the `record` role of your camera. + + +```yaml +camera_name: + gstreamer: + video_format: video/x-h265 + inputs: + - path: rtsp://10.0.0.1:554/stream0 + audio_pipeline: + - rtppcmadepay + - alawdec + - audioconvert + - queue + - voaacenc + roles: + - record + + - path: rtsp://10.0.0.1:554/stream1 + roles: + - detect + detect: + width: 1920 + height: 1080 +``` + +In the example above, `audio_pipeline` has a setup that is equivalent to having the `audio_format: audio/x-alaw` option. + +If you want to disable audio, please set `audio_format: none`. If you specify no audio format for the `record` role, GStreamer integration will run a `gst-discoverer-1.0` for detecting audio format. + + +## Advanced configuration + +If you have a very specific camera and you're handy with the gstreamer, you can use `raw_pipeline` to specify your pipeline. +This will give you full control over gstreamer behavior and allow tweaking and troubleshooting issues. + +Make sure to keep the `roles` array in sync with the behavior of your `raw_pipeline`. For example, do not use `fdsink` for record-only inputs. + + +```yaml +camera_name: + gstreamer: + video_format: video/x-h265 + inputs: + - path: whatever + raw_pipeline: + - rtspsrc location="rtsp://some/url" name=rtp_stream protocols=tcp latency=0 do-timestamp=true + - rtpjitterbuffer do-lost=true drop-on-latency=true + - rtph264depay + - tee name=depayed_stream + - queue + - nvv4l2decoder enable-max-performance=true + - "video/x-raw(memory:NVMM),format=NV12" + - nvvidconv + - "video/x-raw,width=(int)1920,height=(int)1080,format=(string)I420" + - fdsink depayed_stream. + - queue + - h264parse + - splitmuxsink async-finalize=true send-keyframe-requests=true max-size-bytes=0 name=mux muxer=mp4mux location=/tmp/cache/cam_name-gstsplitmuxchunk-%05d.mp4 max-size-time=10000000000 rtp_stream. + - queue + - rtpmp4gdepay + - aacparse + - mux.audio_0 + roles: + - record + - detect + detect: + width: 1920 + height: 1080 +``` + +This pipeline uses NVidia `nvv4l2decoder` with both detect and recording capabilities, using `h264` video and `audio/mpeg` streams. + +This pipeline can be split into multiple blocks. The first block is an input pipeline. It consists of `rtspsrc` and `rtpjitterbuffer` plugins. + +```yaml + - rtspsrc location="rtsp://some/url" name=rtp_stream protocols=tcp latency=0 do-timestamp=true + - rtpjitterbuffer do-lost=true drop-on-latency=true +``` + +This block sets up the `rtsp` source and adds the `rtpjitterbuffer`. `rtpjitterbuffer` dedupes the RTP packets and create a PTS on the outgoing buffer. + +This block extracts H264 video from RTP packets (RFC 3984) + +```yaml + - rtph264depay +``` + +This block splits the H264 video for detection and recording pipelines. +```yaml + - tee name=depayed_stream + - queue +``` + +This block decodes the H264 video stream and outputs it in `I420`. Frigate does require the video to be in `I420` since Frigate uses a grayscale image for motion detection for better performance. +`fdsink` put the output to the /dev/stdout captured by the Frigate. Make sure to keep `fdsink depayed_stream.` The `depayed_stream.` uses the stream from the `tee name=depayed_stream` for the recording. + +```yaml +- nvv4l2decoder enable-max-performance=true +- "video/x-raw(memory:NVMM),format=NV12" +- nvvidconv +- "video/x-raw,width=(int)1920,height=(int)1080,format=(string)I420" +- fdsink depayed_stream. +``` + +This block prepare the H264 video stream for recording and create named `mp4mux` for muxing audio and video streams. The resulting stream will be put into `.mp4` files with a max of 10 seconds in length. The actual length might be between 6 and 8 seconds since the keyframe is used to detect the actual time. +The `rtp_stream.` emits the RTP stream from the `rtspsrc` element. + +```yaml +- queue +- h264parse +- splitmuxsink async-finalize=true send-keyframe-requests=true max-size-bytes=0 name=mux muxer=mp4mux location=/tmp/cache/cam_name-gstsplitmuxchunk-%05d.mp4 max-size-time=10000000000 rtp_stream. +``` + +The last block does the extraction of the audio stream from RTP. Then prepare it for the `mp4mux` block. Since the audio stream, in this case, is `audio/mpeg`, the pipeline only does `aacparse` to extract the encoded audio from the RTP stream. + +```yaml +- queue +- rtpmp4gdepay +- aacparse +- mux.audio_0 +``` + +### Tweaking the standard configuration + +In most cases, you probably do not need to come up with your own GStreamer pipeline. Instead, you may want to tweak some of the blocks to get a better result. +A good example might be an `audio_pipeline` we discussed above. It allows the addition of non-supported audio streams for the recording. +GStreamer integration has the following configuration parameters: + +- raw_pipeline +- input_options +- video_format +- audio_format +- audio_pipeline +- record_pipeline + +We have already discussed some of them. Let's look into `input_options` and `record_pipeline`. + +### Input path and input_options + +Input path provides the URI to the camera stream. `rtsp://` and `rtmp://` schemes are supported. +For each scheme, a correspondent GStreamer pipeline element is used: `rtspsrc` for "rtsp://" and `rtmpsrc` for `rtmp://` + +GStreamer adds `latency=0 do-timestamp=true` parameters for the `rtspsrc`. +However, you might need to add extra arguments. To do that, you can use `input_options` array. + +For instance, standard parameters can be passed this way: + +```yaml +camera_name: + gstreamer: + video_format: video/x-h265 + inputs: + - path: rtsp://10.0.0.1:554/stream0 + input_options: + - latency=0 + - do-timestamp=true + ... +``` + +You can even add a pipeline element right after the `rtspsrc` + +```yaml +camera_name: + gstreamer: + video_format: video/x-h265 + inputs: + - path: rtsp://10.0.0.1:554/stream0 + input_options: + - latency=0 + - do-timestamp=true + - "! rtpjitterbuffer do-lost=true" + ... +``` + +Note the ` ! ` before `rtpjitterbuffer`. It indicates the new pipeline element. + +This setup equivalent to the following raw pipeline snippet: + +```yaml + raw_pipeline: + - rtspsrc location="rtsp://10.0.0.1:554/stream0" latency=0 do-timestamp=true + - rtpjitterbuffer do-lost=true +``` + +You can even completely replace the input pipeline. If integration does not see the `rtsp://` or `rtmp://` in the input patch, +it will consider it as a raw input pipeline, not a path. + +```yaml +camera_name: + gstreamer: + video_format: video/x-h265 + inputs: + - path: srtsrc uri="srt://127.0.0.1:7001" latency=0 name=rtp_stream + roles: + - detect + ... +``` + +This setup allows using a non-supported SRT stream as a source. + +**Note:** SRT stream would not work without tweaking depay and decode elements. The `srtsrc` is mentioned here as an example. + + +### record_pipeline element + +record_pipeline allows replacing the record pipeline for the `record` role. By default `record_pipeline` consists of one gstreamer element - `h264parse` or `h265parse`, depending on the video codec you set up. This setup prevents video re-encoding and saves resources. + +However, for some edge cases, you might find it useful to do some sort of video transformation. You may add some GStreamer video enhancement elements, or even add some ML-based elements, though you need to build a custom build for that. + +## Experimenting with GSreamer + +You might find yourself stuck with a non-working GStreamer pipeline. To troubleshoot it, you can copy the resulting GStreamer pipeline and put it into the bash file to run it separately from GStreamer. +You just need to do a couple of tweaks: + +- Put your URI into the environment variable, such as `LOC="rtsp://user:pwd@0.0.0.0:554/stream0"` That way you won't confuse your shell +- Keep `'video/x-raw(memory:NVMM),format=NV12'` and `'video/x-raw,width=(int)704,height=(int)576,format=(string)I420'` elements inside single quotes. +- Replace `fdsink` with `autovideosink async=true` to get a video overlay +- Replace `location` parameter of the `splitmuxsink` to point to your local folder. + +This is an example script to detect the camera stream: + +``` +LOC="rtsp://user:pass@1.2.3.4:554/stream0" +gst-discoverer-1.0 -v $LOC +``` + + +This is an example script to run video stream with the recording: + +``` +gst-launch-1.0 rtspsrc location=$LOC name=rtp_stream latency=0 do-timestamp=true live=true ! \ + rtpjitterbuffer do-lost=true drop-on-latency=true ! \ + rtph265depay ! \ + tee name=depayed_stream ! \ + queue ! \ + nvv4l2decoder enable-max-performance=true ! \ + 'video/x-raw(memory:NVMM),format=NV12' ! \ + nvvidconv ! \ + 'video/x-raw,width=(int)704,height=(int)576,format=(string)I420' ! \ + autovideosink async=true depayed_stream. ! \ + queue ! \ + h265parse config-interval=-1 ! \ + splitmuxsink name=mux muxer=mp4mux async-handling=true location=loc-gstsplitmuxchunk-%05d.mp4 max-size-time=10000000000 rtp_stream. ! \ + queue ! rtppcmadepay ! alawdec ! audioconvert ! queue ! voaacenc ! mux.audio_0 +``` + diff --git a/docs/docs/configuration/hardware_acceleration.md b/docs/docs/configuration/hardware_acceleration.md index e2153f4f2b..3f1ebc51fb 100644 --- a/docs/docs/configuration/hardware_acceleration.md +++ b/docs/docs/configuration/hardware_acceleration.md @@ -68,3 +68,9 @@ ffmpeg: ### NVIDIA GPU NVIDIA GPU based decoding via NVDEC is supported, but requires special configuration. See the [NVIDIA NVDEC documentation](/configuration/nvdec) for more details. + + +### TensorRT on NVIDIA Jetson family devices + +For the NVIDIA Jetson family devices, you might use both ffmpeg and gstreamer decoders. As of now, ffmpeg does not fully support NVDEC on the Jetson devices. As an alternative, you may use gstreamer with the `nvv4l2decoder` plugin to enable the NVDEC and NVENC features of Jetson devices. +See the See the [GStreamer documentation](/configuration/gstreamer) for more details. \ No newline at end of file diff --git a/docs/docs/hardware.md b/docs/docs/hardware.md index 71fe28dfdc..1f5fa65fce 100644 --- a/docs/docs/hardware.md +++ b/docs/docs/hardware.md @@ -34,6 +34,8 @@ My current favorite is the Odyssey X86 Blue J4125 because the Coral M.2 compatib | Raspberry Pi 3B (32bit) (affiliate link) | 60ms | USB | Can handle a small number of cameras, but the detection speeds are slow due to USB 2.0. | | Raspberry Pi 4 (32bit) (affiliate link) | 15-20ms | USB | Can handle a small number of cameras. The 2GB version runs fine. | | Raspberry Pi 4 (64bit) (affiliate link) | 10-15ms | USB | Can handle a small number of cameras. The 2GB version runs fine. | +| NVIDIA Jetson Nano 4GB | 25-50ms | | With the gstreamer and NVDEC hardware decoding can handle up to 6-8 cameras depending on the resolution and FPS. | +| NVIDIA Jetson Xavier | | | NVIDIA Jetson Xavier is known to be a more powerful device than Nano and should handle more cameras with higher resolution and FPS.| ## Google Coral TPU diff --git a/docs/package.json b/docs/package.json index 0e159d8c36..aca8f347bc 100644 --- a/docs/package.json +++ b/docs/package.json @@ -4,11 +4,11 @@ "private": true, "scripts": { "docusaurus": "docusaurus", - "start": "docusaurus start", + "start": "docusaurus start --host 0.0.0.0", "build": "docusaurus build", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", - "serve": "docusaurus serve", + "serve": "docusaurus serve --host 0.0.0.0", "clear": "docusaurus clear" }, "dependencies": { diff --git a/docs/sidebars.js b/docs/sidebars.js index 2b3669bdff..b0a6310728 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -26,6 +26,7 @@ module.exports = { 'configuration/hardware_acceleration', 'configuration/nvdec', 'configuration/camera_specific', + 'configuration/gstreamer', ], Integrations: ['integrations/home-assistant', 'integrations/api', 'integrations/mqtt'], Troubleshooting: ['faqs'], diff --git a/frigate/app.py b/frigate/app.py index bf593d6ac3..254def07c4 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -17,7 +17,7 @@ from frigate.config import DetectorTypeEnum, FrigateConfig from frigate.const import CACHE_DIR, CLIPS_DIR, RECORD_DIR -from frigate.edgetpu import EdgeTPUProcess +from frigate.detection import DetectionProcess from frigate.events import EventCleanup, EventProcessor from frigate.http import create_app from frigate.log import log_process, root_configurer @@ -40,7 +40,7 @@ def __init__(self): self.base_config: FrigateConfig = None self.config: FrigateConfig = None self.detection_queue = mp.Queue() - self.detectors: Dict[str, EdgeTPUProcess] = {} + self.detectors: Dict[str, DetectionProcess] = {} self.detection_out_events: Dict[str, mp.Event] = {} self.detection_shms: List[mp.shared_memory.SharedMemory] = [] self.log_queue = mp.Queue() @@ -89,7 +89,7 @@ def init_config(self): "detection_fps": mp.Value("d", 0.0), "detection_frame": mp.Value("d", 0.0), "read_start": mp.Value("d", 0.0), - "ffmpeg_pid": mp.Value("i", 0), + "decoder_pid": mp.Value("i", 0), "frame_queue": mp.Queue(maxsize=2), } @@ -182,27 +182,15 @@ def start_detectors(self): self.detection_shms.append(shm_in) self.detection_shms.append(shm_out) - for name, detector in self.config.detectors.items(): - if detector.type == DetectorTypeEnum.cpu: - self.detectors[name] = EdgeTPUProcess( - name, - self.detection_queue, - self.detection_out_events, - model_path, - model_shape, - "cpu", - detector.num_threads, - ) - if detector.type == DetectorTypeEnum.edgetpu: - self.detectors[name] = EdgeTPUProcess( - name, - self.detection_queue, - self.detection_out_events, - model_path, - model_shape, - detector.device, - detector.num_threads, - ) + for name, detector_config in self.config.detectors.items(): + self.detectors[name] = DetectionProcess( + name, + self.detection_queue, + self.detection_out_events, + model_path, + model_shape, + detector_config, + ) def start_detected_frames_processor(self): self.detected_frames_processor = TrackedObjectProcessor( diff --git a/frigate/config.py b/frigate/config.py index a81f3241a5..40c895317c 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -9,11 +9,18 @@ import matplotlib.pyplot as plt import numpy as np import yaml -from pydantic import BaseModel, Extra, Field, validator +from pydantic import BaseModel, Extra, Field, validator, root_validator from pydantic.fields import PrivateAttr -from frigate.const import BASE_DIR, CACHE_DIR, YAML_EXT -from frigate.util import create_mask, deep_merge, load_labels +from frigate.const import BASE_DIR, CACHE_DIR, YAML_EXT, RECORD_SEGMENT_TIME_SECONDS +from frigate.util import ( + create_mask, + deep_merge, + load_labels, + empty_or_none, +) + +from frigate.gstreamer import gst_discover, get_gstreamer_builder logger = logging.getLogger(__name__) @@ -26,7 +33,9 @@ DEFAULT_TRACKED_OBJECTS = ["person"] DEFAULT_DETECTORS = {"cpu": {"type": "cpu"}} - +# TensorRT defaults +DEFAULT_TRT_MODEL_PATH="/yolo4/yolov4-tiny-416.trt" +DEFAULT_TRT_MODEL_SIZE=416 class FrigateBaseModel(BaseModel): class Config: @@ -36,6 +45,7 @@ class Config: class DetectorTypeEnum(str, Enum): edgetpu = "edgetpu" cpu = "cpu" + tensorrt = "tensorrt" class DetectorConfig(FrigateBaseModel): @@ -335,7 +345,7 @@ class BirdseyeConfig(FrigateBaseModel): "-f", "segment", "-segment_time", - "10", + str(RECORD_SEGMENT_TIME_SECONDS), "-segment_format", "mp4", "-reset_timestamps", @@ -379,6 +389,27 @@ class FfmpegConfig(FrigateBaseModel): ) +class GstreamerConfig(FrigateBaseModel): + input_options: List[str] = Field( + default=[], + title="Add additional options to the rtspsrc or even ", + ) + video_format: Optional[str] = Field( + title="A video format of the camera stream. Can be video/x-h265, video/x-h264. If not set, Frigate will try to autodetect.", + ) + audio_format: Optional[str] = Field( + title="An audio format of the camera stream for recording. Supported audio/mpeg and audio/x-alaw. If not set, Frigate will try to autodetect it.", + ) + audio_pipeline: List[str] = Field( + default=[], + title="Custom audio pipeline. Example: rtppcmadepay, alawdec, audioconvert, avenc_aac", + ) + record_pipeline: List[str] = Field( + default=[], + title="Custom pipeline for the recorder. by default it's h265parse or h264parse", + ) + + class CameraRoleEnum(str, Enum): record = "record" rtmp = "rtmp" @@ -388,6 +419,9 @@ class CameraRoleEnum(str, Enum): class CameraInput(FrigateBaseModel): path: str = Field(title="Camera input path.") roles: List[CameraRoleEnum] = Field(title="Roles assigned to this input.") + + +class CameraFFmpegInput(CameraInput): global_args: Union[str, List[str]] = Field( default_factory=list, title="FFmpeg global arguments." ) @@ -399,21 +433,58 @@ class CameraInput(FrigateBaseModel): ) +class CameraGStreamerInput(CameraInput): + raw_pipeline: List[str] = Field( + default=[], + title="Override full pipeline. The pipeline should start with the arguments after the `gst-launch-1.0`, `-q`", + ) + input_options: List[str] = Field( + default=[], + title="Add additional options to the rtspsrc or even ", + ) + video_format: Optional[str] = Field( + title="A video format of the camera stream. Can be video/x-h265, video/x-h264. If not set, Frigate will try to autodetect.", + ) + audio_format: Optional[str] = Field( + title="An audio format of the camera stream for recording. Supported audio/mpeg and audio/x-alaw. If not set, Frigate will try to autodetect it.", + ) + audio_pipeline: List[str] = Field( + default=[], + title="Custom audio pipeline. Example: rtppcmadepay, alawdec, audioconvert, avenc_aac", + ) + record_pipeline: List[str] = Field( + default=[], + title="Custom pipeline for the recorder. by default it's h265parse or h264parse", + ) + + +def validate_roles(cls, v): + roles = [role for i in v for role in i.roles] + roles_set = set(roles) + + if len(roles) > len(roles_set): + raise ValueError("Each input role may only be used once.") + + if not "detect" in roles: + raise ValueError("The detect role is required.") + + return v + + class CameraFfmpegConfig(FfmpegConfig): - inputs: List[CameraInput] = Field(title="Camera inputs.") + inputs: List[CameraFFmpegInput] = Field(title="Camera FFMpeg inputs.") @validator("inputs") def validate_roles(cls, v): - roles = [role for i in v for role in i.roles] - roles_set = set(roles) + return validate_roles(cls, v) - if len(roles) > len(roles_set): - raise ValueError("Each input role may only be used once.") - if not "detect" in roles: - raise ValueError("The detect role is required.") +class CameraGStreamerConfig(GstreamerConfig): + inputs: List[CameraGStreamerInput] = Field(title="Camera GStreamer inputs.") - return v + @validator("inputs") + def validate_roles(cls, v): + return validate_roles(cls, v) class SnapshotsConfig(FrigateBaseModel): @@ -501,7 +572,10 @@ class CameraLiveConfig(FrigateBaseModel): class CameraConfig(FrigateBaseModel): name: Optional[str] = Field(title="Camera name.", regex="^[a-zA-Z0-9_-]+$") - ffmpeg: CameraFfmpegConfig = Field(title="FFmpeg configuration for the camera.") + ffmpeg: Optional[CameraFfmpegConfig] = Field(title="FFmpeg configuration for the camera.") + gstreamer: Optional[CameraGStreamerConfig] = Field( + title="GStreamer configuration for the camera." + ) best_image_timeout: int = Field( default=60, title="How long to wait for the image with the highest confidence score.", @@ -534,7 +608,7 @@ class CameraConfig(FrigateBaseModel): timestamp_style: TimestampStyleConfig = Field( default_factory=TimestampStyleConfig, title="Timestamp style configuration." ) - _ffmpeg_cmds: List[Dict[str, List[str]]] = PrivateAttr() + _decoder_cmds: List[Dict[str, List[str]]] = PrivateAttr() def __init__(self, **config): # Set zone colors @@ -546,8 +620,9 @@ def __init__(self, **config): } # add roles to the input if there is only one - if len(config["ffmpeg"]["inputs"]) == 1: - config["ffmpeg"]["inputs"][0]["roles"] = ["record", "rtmp", "detect"] + if "ffmpeg" in config: + if len(config["ffmpeg"]["inputs"]) == 1: + config["ffmpeg"]["inputs"][0]["roles"] = ["record", "rtmp", "detect"] super().__init__(**config) @@ -560,22 +635,105 @@ def frame_shape_yuv(self) -> Tuple[int, int]: return self.detect.height * 3 // 2, self.detect.width @property - def ffmpeg_cmds(self) -> List[Dict[str, List[str]]]: - return self._ffmpeg_cmds + def decoder_cmds(self) -> List[Dict[str, List[str]]]: + return self._decoder_cmds - def create_ffmpeg_cmds(self): - if "_ffmpeg_cmds" in self: + def create_decoder_cmds(self): + if "_decoder_cmds" in self: return - ffmpeg_cmds = [] - for ffmpeg_input in self.ffmpeg.inputs: - ffmpeg_cmd = self._get_ffmpeg_cmd(ffmpeg_input) - if ffmpeg_cmd is None: - continue + self._decoder_cmds = [] + if self.ffmpeg: + for ffmpeg_input in self.ffmpeg.inputs: + ffmpeg_cmd = self._get_ffmpeg_cmd(ffmpeg_input) + if ffmpeg_cmd is None: + continue + + self._decoder_cmds.append( + {"roles": ffmpeg_input.roles, "cmd": ffmpeg_cmd} + ) + else: + for input in self.gstreamer.inputs: + gst_cmd = self._get_gstreamer_cmd(self.gstreamer, input) + logger.info("gstreamer command[%s] %s", self.name, gst_cmd) + self._decoder_cmds.append({"roles": input.roles, "cmd": gst_cmd}) + + def _get_gstreamer_cmd( + self, + base_config: GstreamerConfig, + gstreamer_input: CameraGStreamerInput, + ): + if CameraRoleEnum.rtmp.value in gstreamer_input.roles: + raise ValueError( + f"{CameraRoleEnum.rtmp.value} role does not supported for the GStreamer integration" + ) + if not empty_or_none(gstreamer_input.raw_pipeline): + logger.warn("You are using raw pipeline for `%s` camera", self.name) + pipeline_args = [ + f"{item} !".split(" ") + for item in gstreamer_input.raw_pipeline + if len(item) > 0 + ] + pipeline_args = [item for sublist in pipeline_args for item in sublist] + return ["gst-launch-1.0", "-q", *pipeline_args][:-1] + + # Get camera configuration. Input congig override the camera config + input_options = ( + base_config.input_options + if empty_or_none(gstreamer_input.input_options) + else gstreamer_input.input_options + ) + video_format = ( + base_config.video_format + if empty_or_none(gstreamer_input.video_format) + else gstreamer_input.video_format + ) + audio_format = ( + base_config.audio_format + if empty_or_none(gstreamer_input.audio_format) + else gstreamer_input.audio_format + ) + audio_pipeline = ( + base_config.audio_pipeline + if empty_or_none(gstreamer_input.audio_pipeline) + else gstreamer_input.audio_pipeline + ) + record_pipeline = ( + base_config.record_pipeline + if empty_or_none(gstreamer_input.record_pipeline) + else gstreamer_input.record_pipeline + ) - ffmpeg_cmds.append({"roles": ffmpeg_input.roles, "cmd": ffmpeg_cmd}) - self._ffmpeg_cmds = ffmpeg_cmds + use_record = CameraRoleEnum.record.value in gstreamer_input.roles + use_detect = CameraRoleEnum.detect.value in gstreamer_input.roles - def _get_ffmpeg_cmd(self, ffmpeg_input: CameraInput): + # run gst_discover if no video format set or no audio format / pipeline set for recording role + run_gst_discover = empty_or_none(video_format) + if use_record: + if base_config.audio_format is None or empty_or_none( + base_config.audio_pipeline + ): + run_gst_discover = True + + caps = {} + if run_gst_discover: + caps = gst_discover( + gstreamer_input.path, self.name, tuple(["width", "height", "video", "audio"]) + ) + + builder = ( + get_gstreamer_builder(self.detect.width, self.detect.height, self.name) + .with_source(gstreamer_input.path, input_options) + .with_video_format(video_format or caps.get("video")) + .with_record_pipeline(record_pipeline) + ) + if audio_pipeline: + builder = builder.with_audio_pipeline(audio_pipeline) + else: + builder = builder.with_audio_format(audio_format or caps.get("audio")) + + return builder.build(use_detect, use_record) + + def _get_ffmpeg_cmd(self, ffmpeg_input: CameraFFmpegInput): ffmpeg_output_args = [] if "detect" in ffmpeg_input.roles: detect_args = ( @@ -645,6 +803,12 @@ def _get_ffmpeg_cmd(self, ffmpeg_input: CameraInput): return [part for part in cmd if part != ""] + @root_validator + def either_ffmpeg_or_gstreamer(cls, v): + if ("ffmpeg" not in v) and ("gstreamer" not in v): + raise ValueError("either ffmpeg or gstreamer should be set") + return v + class DatabaseConfig(FrigateBaseModel): path: str = Field( @@ -764,7 +928,7 @@ def runtime_config(self) -> FrigateConfig: if config.mqtt.password: config.mqtt.password = config.mqtt.password.format(**FRIGATE_ENV_VARS) - # Global config to propegate down to camera level + # Global config to propagate down to camera level global_config = config.dict( include={ "record": ..., @@ -797,8 +961,9 @@ def runtime_config(self) -> FrigateConfig: camera_config.detect.stationary.threshold = stationary_threshold # FFMPEG input substitution - for input in camera_config.ffmpeg.inputs: - input.path = input.path.format(**FRIGATE_ENV_VARS) + if "ffmpeg" in camera_config: + for input in camera_config.ffmpeg.inputs: + input.path = input.path.format(**FRIGATE_ENV_VARS) # Add default filters object_keys = camera_config.objects.track @@ -844,8 +1009,13 @@ def runtime_config(self) -> FrigateConfig: ) # check runtime config + decoder_config = ( + camera_config.ffmpeg + if camera_config.ffmpeg is not None + else camera_config.gstreamer + ) assigned_roles = list( - set([r for i in camera_config.ffmpeg.inputs for r in i.roles]) + set([r for i in decoder_config.inputs for r in i.roles]) ) if camera_config.record.enabled and not "record" in assigned_roles: raise ValueError( @@ -879,10 +1049,25 @@ def runtime_config(self) -> FrigateConfig: logger.warning( f"{name}: Recording retention is configured for {camera_config.record.retain.mode} and event retention is configured for {camera_config.record.events.retain.mode}. The more restrictive retention policy will be applied." ) - # generage the ffmpeg commands - camera_config.create_ffmpeg_cmds() + # generage the decoder commands + camera_config.create_decoder_cmds() config.cameras[name] = camera_config + for name, detector_config in config.detectors.items(): + if ( + detector_config.type == DetectorTypeEnum.tensorrt + ): + if config.model.path is None: + logger.info( + "Setting default model to the yolov4-tiny-416 for the %s detector.", + name, + ) + config.model.path = DEFAULT_TRT_MODEL_PATH + config.model.height = DEFAULT_TRT_MODEL_SIZE + config.model.width = DEFAULT_TRT_MODEL_SIZE + elif "tflite" in config.model.path: + raise ValueError(f"The {name} detector is of type tensorrt, however, tflite model is used.") + return config @validator("cameras") diff --git a/frigate/const.py b/frigate/const.py index afb6075a7a..22e50c46e2 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -3,3 +3,5 @@ RECORD_DIR = f"{BASE_DIR}/recordings" CACHE_DIR = "/tmp/cache" YAML_EXT = (".yaml", ".yml") +RECORD_SEGMENT_TIME_SECONDS = 10 +GSTREAMER_RECORD_SUFFIX = "-gstsplitmuxchunk" diff --git a/frigate/detection/__init__.py b/frigate/detection/__init__.py new file mode 100644 index 0000000000..c77626b646 --- /dev/null +++ b/frigate/detection/__init__.py @@ -0,0 +1,210 @@ +import datetime +import logging +import multiprocessing as mp +import os +import queue +import signal +import threading +import os +import numpy as np +import multiprocessing as mp +from frigate.util import EventsPerSecond, SharedMemoryFrameManager, listen +from frigate.config import DetectorConfig, DetectorTypeEnum +from frigate.detection.object_detector import ObjectDetector +import importlib +from setproctitle import setproctitle +from typing import Dict, Callable + + +logger = logging.getLogger(__name__) + + +DETECTORS = { + DetectorTypeEnum.cpu: "edgetpu", + DetectorTypeEnum.edgetpu: "edgetpu", + DetectorTypeEnum.tensorrt: "tensorrt", +} + + +def get_object_detector_factory( + detector_config: DetectorConfig, model_path: str +) -> Callable[[], ObjectDetector]: + """ + Return an object detector factory. + Since resource initialization might be performed on python import, + delay module load until the thread started + """ + detector_module = DETECTORS.get(detector_config.type) + if detector_module is None: + logger.error(f"Unsupported detector type '{detector_config.type}'.") + return None + + def _detector_factory() -> ObjectDetector: + path = os.path.join(os.path.dirname(__file__), f"{detector_module}.py") + spec = importlib.util.spec_from_file_location( + f"frigate.detection.{detector_module}", path + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + object_detector = module.object_detector_factory(detector_config, model_path) + return object_detector + + return _detector_factory + + +def run_detector( + name: str, + detection_queue: mp.Queue, + out_events: Dict[str, mp.Event], + avg_speed, + start, + model_shape, + object_detector_factory: Callable[[], ObjectDetector], +): + threading.current_thread().name = f"detector:{name}" + logger = logging.getLogger(f"detector.{name}") + logger.info(f"Starting detection process: {os.getpid()}") + setproctitle(f"frigate.detector.{name}") + listen() + + stop_event = mp.Event() + + def receiveSignal(signalNumber, frame): + stop_event.set() + + signal.signal(signal.SIGTERM, receiveSignal) + signal.signal(signal.SIGINT, receiveSignal) + + frame_manager = SharedMemoryFrameManager() + + outputs = {} + for name in out_events.keys(): + out_shm = mp.shared_memory.SharedMemory(name=f"out-{name}", create=False) + out_np = np.ndarray((20, 6), dtype=np.float32, buffer=out_shm.buf) + outputs[name] = {"shm": out_shm, "np": out_np} + + object_detector = object_detector_factory() + while not stop_event.is_set(): + try: + connection_id = detection_queue.get(timeout=5) + except queue.Empty: + continue + input_frame = frame_manager.get( + connection_id, (model_shape[0], model_shape[1], 3) + ) + + if input_frame is None: + continue + + # detect and send the output + start.value = datetime.datetime.now().timestamp() + detections = object_detector.detect_raw(input_frame) + duration = datetime.datetime.now().timestamp() - start.value + outputs[connection_id]["np"][:] = detections[:] + out_events[connection_id].set() + start.value = 0.0 + + avg_speed.value = (avg_speed.value * 9 + duration) / 10 + del object_detector + logger.debug("Object detector process[%s] exit.", name) + + +class DetectionProcess: + def __init__( + self, + name, + detection_queue, + out_events, + model_path, + model_shape, + detector_config: DetectorConfig, + ): + self.name = name + self.out_events = out_events + self.detection_queue = detection_queue + self.avg_inference_speed = mp.Value("d", 0.01) + self.detection_start = mp.Value("d", 0.0) + self.detect_process = None + self.model_path = model_path + self.model_shape = model_shape + self.detector_config = detector_config + + self.object_detector_factory = get_object_detector_factory( + detector_config, model_path + ) + if self.object_detector_factory: + self.start_or_restart() + + def stop(self): + self.detect_process.terminate() + logging.info("Waiting for detection process to exit gracefully...") + self.detect_process.join(timeout=30) + if self.detect_process.exitcode is None: + logging.info("Detection process didnt exit. Force killing...") + self.detect_process.kill() + self.detect_process.join() + + def start_or_restart(self): + self.detection_start.value = 0.0 + if (not self.detect_process is None) and self.detect_process.is_alive(): + self.stop() + self.detect_process = mp.Process( + target=run_detector, + name=f"detector:{self.name}", + args=( + self.name, + self.detection_queue, + self.out_events, + self.avg_inference_speed, + self.detection_start, + self.model_shape, + self.object_detector_factory, + ), + ) + self.detect_process.daemon = True + self.detect_process.start() + + +class RemoteObjectDetector: + def __init__(self, name, labels, detection_queue, event, model_shape): + self.labels = labels + self.name = name + self.fps = EventsPerSecond() + self.detection_queue = detection_queue + self.event = event + self.shm = mp.shared_memory.SharedMemory(name=self.name, create=False) + self.np_shm = np.ndarray( + (1, model_shape[0], model_shape[1], 3), dtype=np.uint8, buffer=self.shm.buf + ) + self.out_shm = mp.shared_memory.SharedMemory( + name=f"out-{self.name}", create=False + ) + self.out_np_shm = np.ndarray((20, 6), dtype=np.float32, buffer=self.out_shm.buf) + + def detect(self, tensor_input, threshold=0.4): + detections = [] + + # copy input to shared memory + self.np_shm[:] = tensor_input[:] + self.event.clear() + self.detection_queue.put(self.name) + result = self.event.wait(timeout=10.0) + + # if it timed out + if result is None: + return detections + + for d in self.out_np_shm: + if d[1] < threshold: + break + label_key = int(d[0]) + if label_key in self.labels: + detections.append( + (self.labels[label_key], float(d[1]), (d[2], d[3], d[4], d[5])) + ) + self.fps.update() + return detections + + def cleanup(self): + self.shm.unlink() + self.out_shm.unlink() diff --git a/frigate/detection/edgetpu.py b/frigate/detection/edgetpu.py new file mode 100644 index 0000000000..86d75f4ae8 --- /dev/null +++ b/frigate/detection/edgetpu.py @@ -0,0 +1,122 @@ +import logging +import multiprocessing as mp +import os +import queue +import signal +import threading +from frigate.config import DetectorConfig, DetectorTypeEnum +from typing import Dict + +import numpy as np + +import tflite_runtime.interpreter as tflite + + +from tflite_runtime.interpreter import load_delegate + +from frigate.util import EventsPerSecond +from .object_detector import ObjectDetector + +logger = logging.getLogger(__name__) + + +def object_detector_factory(detector_config: DetectorConfig, model_path: str): + if not ( + detector_config.type == DetectorTypeEnum.cpu + or detector_config.type == DetectorTypeEnum.edgetpu + ): + return None + object_detector = LocalObjectDetector( + tf_device=detector_config.type, + model_path=model_path, + num_threads=detector_config.num_threads, + ) + return object_detector + + +class LocalObjectDetector(ObjectDetector): + def __init__(self, tf_device=None, model_path=None, num_threads=3): + self.fps = EventsPerSecond() + # TODO: process_clip + # if labels is None: + # self.labels = {} + # else: + # self.labels = load_labels(labels) + + device_config = {"device": "usb"} + if not tf_device is None: + device_config = {"device": tf_device} + + edge_tpu_delegate = None + + if tf_device != "cpu": + try: + logger.info(f"Attempting to load TPU as {device_config['device']}") + edge_tpu_delegate = load_delegate("libedgetpu.so.1.0", device_config) + logger.info("TPU found") + self.interpreter = tflite.Interpreter( + model_path=model_path or "/edgetpu_model.tflite", + experimental_delegates=[edge_tpu_delegate], + ) + except ValueError: + logger.error( + "No EdgeTPU was detected. If you do not have a Coral device yet, you must configure CPU detectors." + ) + raise + else: + logger.warning( + "CPU detectors are not recommended and should only be used for testing or for trial purposes." + ) + self.interpreter = tflite.Interpreter( + model_path=model_path or "/cpu_model.tflite", num_threads=num_threads + ) + + self.interpreter.allocate_tensors() + + self.tensor_input_details = self.interpreter.get_input_details() + self.tensor_output_details = self.interpreter.get_output_details() + + def detect(self, tensor_input, threshold=0.4): + # TODO: process_clip + detections = [] + + raw_detections = self.detect_raw(tensor_input) + + for d in raw_detections: + if d[1] < threshold: + break + detections.append( + (self.labels[int(d[0])], float(d[1]), (d[2], d[3], d[4], d[5])) + ) + self.fps.update() + return detections + + def detect_raw(self, tensor_input): + # Expand dimensions [height, width, 3] ince the model expects images to have shape [1, height, width, 3] + tensor_input = np.expand_dims(tensor_input, axis=0) + + self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input) + self.interpreter.invoke() + + boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0] + class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0] + scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0] + count = int( + self.interpreter.tensor(self.tensor_output_details[3]["index"])()[0] + ) + + detections = np.zeros((20, 6), np.float32) + + for i in range(count): + if scores[i] < 0.4 or i == 20: + break + detections[i] = [ + class_ids[i], + float(scores[i]), + boxes[i][0], + boxes[i][1], + boxes[i][2], + boxes[i][3], + ] + + return detections diff --git a/frigate/detection/object_detector.py b/frigate/detection/object_detector.py new file mode 100644 index 0000000000..d12569b6b2 --- /dev/null +++ b/frigate/detection/object_detector.py @@ -0,0 +1,7 @@ +from abc import ABC, abstractmethod + + +class ObjectDetector(ABC): + @abstractmethod + def detect(self, tensor_input, threshold=0.4): + pass diff --git a/frigate/detection/tensorrt.py b/frigate/detection/tensorrt.py new file mode 100644 index 0000000000..3791000c92 --- /dev/null +++ b/frigate/detection/tensorrt.py @@ -0,0 +1,223 @@ +import logging +from frigate.config import DetectorConfig, DetectorTypeEnum +from frigate.util import EventsPerSecond +import ctypes +import numpy as np +import tensorrt as trt +import pycuda.driver as cuda +from .object_detector import ObjectDetector +import pycuda.autoinit # This is needed for initializing CUDA driver + +logger = logging.getLogger(__name__) + + +def object_detector_factory(detector_config: DetectorConfig, model_path: str): + if detector_config.type != DetectorTypeEnum.tensorrt: + return None + try: + ctypes.cdll.LoadLibrary("/yolo4/libyolo_layer.so") + except OSError as e: + logger.error("ERROR: failed to load /yolo4/libyolo_layer.so. %s", e) + return LocalObjectDetector(detector_config, model_path) + + +class HostDeviceMem(object): + """Simple helper data class that's a little nicer to use than a 2-tuple.""" + + def __init__(self, host_mem, device_mem): + self.host = host_mem + self.device = device_mem + + def __str__(self): + return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device) + + def __repr__(self): + return self.__str__() + + +class LocalObjectDetector(ObjectDetector): + def _load_engine(self, model_path): + with open(model_path, "rb") as f, trt.Runtime(self.trt_logger) as runtime: + return runtime.deserialize_cuda_engine(f.read()) + + def _get_input_shape(self): + """Get input shape of the TensorRT YOLO engine.""" + binding = self.engine[0] + assert self.engine.binding_is_input(binding) + binding_dims = self.engine.get_binding_shape(binding) + if len(binding_dims) == 4: + return tuple(binding_dims[2:]) + elif len(binding_dims) == 3: + return tuple(binding_dims[1:]) + else: + raise ValueError( + "bad dims of binding %s: %s" % (binding, str(binding_dims)) + ) + + def _allocate_buffers(self): + """Allocates all host/device in/out buffers required for an engine.""" + inputs = [] + outputs = [] + bindings = [] + output_idx = 0 + stream = cuda.Stream() + for binding in self.engine: + binding_dims = self.engine.get_binding_shape(binding) + if len(binding_dims) == 4: + # explicit batch case (TensorRT 7+) + size = trt.volume(binding_dims) + elif len(binding_dims) == 3: + # implicit batch case (TensorRT 6 or older) + size = trt.volume(binding_dims) * self.engine.max_batch_size + else: + raise ValueError( + "bad dims of binding %s: %s" % (binding, str(binding_dims)) + ) + dtype = trt.nptype(self.engine.get_binding_dtype(binding)) + # Allocate host and device buffers + host_mem = cuda.pagelocked_empty(size, dtype) + device_mem = cuda.mem_alloc(host_mem.nbytes) + # Append the device buffer to device bindings. + bindings.append(int(device_mem)) + # Append to the appropriate list. + if self.engine.binding_is_input(binding): + inputs.append(HostDeviceMem(host_mem, device_mem)) + else: + # each grid has 3 anchors, each anchor generates a detection + # output of 7 float32 values + assert size % 7 == 0 + outputs.append(HostDeviceMem(host_mem, device_mem)) + output_idx += 1 + assert len(inputs) == 1 + assert len(outputs) == 1 + return inputs, outputs, bindings, stream + + def _do_inference(self): + """do_inference (for TensorRT 7.0+) + + This function is generalized for multiple inputs/outputs for full + dimension networks. + Inputs and outputs are expected to be lists of HostDeviceMem objects. + """ + # Transfer input data to the GPU. + [ + cuda.memcpy_htod_async(inp.device, inp.host, self.stream) + for inp in self.inputs + ] + # Run inference. + self.context.execute_async_v2( + bindings=self.bindings, stream_handle=self.stream.handle + ) + # Transfer predictions back from the GPU. + [ + cuda.memcpy_dtoh_async(out.host, out.device, self.stream) + for out in self.outputs + ] + # Synchronize the stream + self.stream.synchronize() + # Return only the host outputs. + return [out.host for out in self.outputs] + + def __init__(self, detector_config: DetectorConfig, model_path: str): + self.fps = EventsPerSecond() + self.conf_th = 0.4 ##TODO: model config parameter + self.nms_threshold = 0.4 + self.trt_logger = trt.Logger(trt.Logger.INFO) + self.engine = self._load_engine(model_path) + self.input_shape = self._get_input_shape() + + try: + self.context = self.engine.create_execution_context() + ( + self.inputs, + self.outputs, + self.bindings, + self.stream, + ) = self._allocate_buffers() + except Exception as e: + logger.error(e) + raise RuntimeError("fail to allocate CUDA resources") from e + + logger.debug("TensorRT loaded. Input shape is %s", self.input_shape) + logger.debug("TensorRT version is %s", trt.__version__[0]) + + def __del__(self): + """Free CUDA memories.""" + del self.outputs + del self.inputs + del self.stream + + def _postprocess_yolo(self, trt_outputs, img_w, img_h, conf_th, nms_threshold): + """Postprocess TensorRT outputs. + + # Args + trt_outputs: a list of 2 or 3 tensors, where each tensor + contains a multiple of 7 float32 numbers in + the order of [x, y, w, h, box_confidence, class_id, class_prob] + conf_th: confidence threshold + + # Returns + boxes, scores, classes + """ + # filter low-conf detections and concatenate results of all yolo layers + detections = [] + for o in trt_outputs: + dets = o.reshape((-1, 7)) + dets = dets[dets[:, 4] * dets[:, 6] >= conf_th] + detections.append(dets) + detections = np.concatenate(detections, axis=0) + + return detections + + def detect(self, tensor_input, threshold=0.4): + pass + + def detect_raw(self, tensor_input): + # Input tensor has the shape of the [height, width, 3] + # Output tensor of float32 of shape [20, 6] where: + # O - class id + # 1 - score + # 2..5 - a value between 0 and 1 of the box: [top, left, bottom, right] + + # transform [height, width, 3] into (3, H, W) + tensor_input = tensor_input.transpose((2, 0, 1)).astype(np.float32) + + # normalize + tensor_input /= 255.0 + + self.inputs[0].host = np.ascontiguousarray(tensor_input) + trt_outputs = self._do_inference() + + raw_detections = self._postprocess_yolo( + trt_outputs, + tensor_input.shape[1], + tensor_input.shape[0], + self.conf_th, + nms_threshold=self.nms_threshold, + ) + + if len(raw_detections) == 0: + return np.zeros((20, 6), np.float32) + + # raw_detections: Nx7 numpy arrays of + # [[x, y, w, h, box_confidence, class_id, class_prob], + + # Calculate score as box_confidence x class_prob + raw_detections[:, 4] = raw_detections[:, 4] * raw_detections[:, 6] + # Reorder elements by the score, best on top, remove class_prob + ordered = raw_detections[raw_detections[:, 4].argsort()[::-1]][:, 0:6] + # transform width to right with clamp to 0..1 + ordered[:, 2] = np.clip(ordered[:, 2] + ordered[:, 0], 0, 1) + # transform height to bottom with clamp to 0..1 + ordered[:, 3] = np.clip(ordered[:, 3] + ordered[:, 1], 0, 1) + # put result into the correct order and limit to top 20 + detections = ordered[:, [5, 4, 1, 0, 3, 2]][:20] + # pad to 20x6 shape + append_cnt = 20 - len(detections) + if append_cnt > 0: + detections = np.append( + detections, np.zeros((append_cnt, 6), np.float32), axis=0 + ) + + self.fps.update() + return detections diff --git a/frigate/gstreamer.py b/frigate/gstreamer.py new file mode 100644 index 0000000000..60e0c27a0f --- /dev/null +++ b/frigate/gstreamer.py @@ -0,0 +1,358 @@ +import re +from functools import lru_cache +import os +import logging +import traceback +import subprocess as sp +from typing import Dict, List, Optional + +from frigate.const import ( + CACHE_DIR, + GSTREAMER_RECORD_SUFFIX, + RECORD_SEGMENT_TIME_SECONDS, +) + +VIDEO_CODEC_CAP_NAME = "video codec" + +logger = logging.getLogger(__name__) + + +@lru_cache +def gst_discover( + source: str, cam_name: str, keys: List[str] +) -> Optional[Dict[str, str]]: + """ + run gst-discoverer-1.0 to discover source stream + and extract keys, specified in the source arrat + """ + try: + data = sp.check_output( + [ + "gst-discoverer-1.0", + "-v", + source, + ], + universal_newlines=True, + start_new_session=True, + stderr=None, + timeout=15, + ) + stripped = list(map(lambda s: s.strip().partition(":"), data.split("\n"))) + result = {} + for key, _, value in stripped: + for param in keys: + if param == key.lower(): + terms = value.strip().split(" ") + result[param] = terms[0].split(",")[0] + return result + except sp.TimeoutExpired: + logger.error( + ( + "gst-discoverer-1.0 timed out auto discovering camera %s. " + "Try setting up `decoder_pipeline` according to your camera video codec." + ), + cam_name, + ) + return {} + except: + logger.error( + "gst-discoverer-1.0 failed with the message: %s", traceback.format_exc() + ) + return {} + + +@lru_cache +def gst_inspect_find_codec(codec: Optional[str]) -> List[str]: + """ + run gst-inspect-1.0 and find the codec. + gst-inspect-1.0 return data in the following format: + omx: omxh265dec: OpenMAX H.265 Video Decoder + rtp: rtph265pay: RTP H265 payloader + """ + try: + data = sp.check_output( + ["gst-inspect-1.0"], + universal_newlines=True, + start_new_session=True, + stderr=None, + ) + data = [ + line.split(":") + for line in data.split("\n") + if codec is None or codec in line + ] + return [item[1].strip() for item in data if len(item) > 1] + except: + logger.error( + "gst-inspect-1.0 failed with the message: %s", traceback.format_exc() + ) + return None + + +RTP_STREAM_NAME_KEY = "name=" +RTP_STREAM_NAME = "rtp_stream" +DEPAYED_STREAM_NAME = "depayed_stream" + + +AUDIO_PIPELINES = { + "audio/mpeg": ["rtpmp4gdepay", "aacparse"], + "audio/x-alaw": ["rtppcmadepay", "alawdec", "audioconvert", "queue", "voaacenc"], +} + + +class GstreamerBaseBuilder: + def __init__(self, width, height, name, format="I420") -> None: + self.width = width + self.height = height + self.name = name + self.format = format + self.input_pipeline = None + self.video_format = None + self.record_pipeline = None + self.audio_pipeline = None + self.raw_pipeline = None + + def with_raw_pipeline(self, raw_pipeline: List[str]): + """ + Set the raw pipeline + """ + self.raw_pipeline = raw_pipeline + return self + + def with_source(self, uri: str, options: List[str]): + """ + Set RTMP or RTSP data source with the list of options + """ + is_rtsp = re.match(r"rtsps?:\/\/", uri) + is_rtmp = re.match(r"rtm(p|pt|ps|pe|fp|pte|pts)?:\/\/", uri) + if is_rtsp: + self.input_pipeline = f'rtspsrc location="{uri}"' + elif is_rtmp: + self.input_pipeline = f'rtmpsrc location="{uri}"' + else: + logger.warning( + "An input url does not start with rtsp:// or rtmp:// for camera %s. Assuming a full input pipeline supplied.", + self.name, + ) + self.input_pipeline = self._to_array(uri) + return self + + has_options = options is not None and len(options) > 0 + extra_options = None + + if has_options: + extra_options = " ".join(options) + if RTP_STREAM_NAME_KEY not in extra_options: + extra_options = ( + f"{RTP_STREAM_NAME_KEY}{RTP_STREAM_NAME} {extra_options}" + ) + else: + extra_options = f"{RTP_STREAM_NAME_KEY}{RTP_STREAM_NAME}" + if is_rtsp: + extra_options = extra_options + " latency=0 do-timestamp=true" + + self.input_pipeline = self._to_array(f"{self.input_pipeline} {extra_options}") + return self + + def with_video_format(self, format: str): + """ + set encoding format. Encoding format should be one of: + h265, h264, h236, h261 or be like `video/x-h265` + """ + if not format: + return self + format = format.lower().replace("video/x-", "") + self.video_format = format + return self + + def with_audio_format(self, format): + """ + set the audio format and make the audio_pipeline + """ + if not format: + return self + + if format in AUDIO_PIPELINES: + self.audio_pipeline = AUDIO_PIPELINES.get(format) + else: + logger.warning("No pipeline set for the '%s' audio format.", format) + return self + + def with_record_pipeline(self, pipeline): + """ + set record pipeline. by default record_pipeline is empty. The splitmuxsink will get the + depayed camera stream and mux it using mp4mux into the file. That way no re-encoding will be performed. + If your camera has a different endcoding format which is not supported by the browser player, + add the record_pipeline to decode and endode the video stream + """ + if pipeline: + self.record_pipeline = pipeline + return self + + def with_audio_pipeline(self, pipeline): + """ + set set the optional audio pipeline to mux audio into the recording. + """ + self.audio_pipeline = pipeline + return self + + @staticmethod + def accept(plugins: List[str]) -> bool: + """ + Accept method receives a list of plugins and return true if the builder can hande the current list + Builder should check all necessary pluguns before returning True + """ + return True + + def _to_array(self, input): + return list(map((lambda el: el.strip()), input.split("!"))) + + def _build_gst_pipeline( + self, pipeline: List[str], use_detect=True, use_record=False + ): + fd_sink = ( + [f"fdsink {DEPAYED_STREAM_NAME}."] + if use_record and use_detect + else (["fdsink"] if use_detect else []) + ) + + record_pipeline = ( + [f"{self.video_format}parse"] + if self.record_pipeline is None + else self.record_pipeline + ) + + use_audio_pipeline = use_record and ( + self.audio_pipeline is not None and len(self.audio_pipeline) > 0 + ) + + split_mux = f"splitmuxsink async-finalize=true send-keyframe-requests=true max-size-bytes=0 " + + if use_audio_pipeline: + split_mux = split_mux + "name=mux muxer=mp4mux " + split_mux = split_mux + ( + f"location={os.path.join(CACHE_DIR, self.name)}{GSTREAMER_RECORD_SUFFIX}-%05d.mp4 " + f"max-size-time={RECORD_SEGMENT_TIME_SECONDS*1000000000}" + ) + + audio_pipeline = [] + if use_audio_pipeline: + # add the RTP stream after the splitmuxsink + split_mux = f"{split_mux} {RTP_STREAM_NAME}." + # add a queue after the rtp_stream. and mux.audio_0 as a receiver + audio_pipeline = ["queue", *self.audio_pipeline, "mux.audio_0"] + + record_mux = ( + [ + "queue", + *record_pipeline, + split_mux, + *audio_pipeline, + ] + if use_record + else [] + ) + + full_pipeline = [*pipeline, *fd_sink, *record_mux] + return full_pipeline + + def _get_default_pipeline(self): + """ + Get a pipeline to render a video test stream + """ + pipeline = [ + "videotestsrc pattern=19", + f"video/x-raw,width=(int){self.width},height=(int){self.height},format=(string){self.format},framerate=20/1", + "videorate drop-only=true", + "video/x-raw,framerate=1/10", + ] + return pipeline + + def get_detect_decoder_pipeline(self) -> List[str]: + return [] + + def _build(self, use_detect: bool, use_record: bool) -> List[str]: + """ + Build a pipeline based on the provided parameters + """ + if self.video_format is None or len(self.video_format) == 0: + return self._build_gst_pipeline( + self._get_default_pipeline(), use_detect=True, use_record=False + ) + depay_element = f"rtp{self.video_format}depay" + + # add rtpjitterbuffer into the input pipeline for reord role if no rtpjitterbuffer has been added already + if use_record: + has_rtpjitterbuffer = "rtpjitterbuffer" in " ".join(self.input_pipeline) + if not has_rtpjitterbuffer: + self.input_pipeline.append( + "rtpjitterbuffer do-lost=true drop-on-latency=true" + ) + + pipeline = [*self.input_pipeline, depay_element] + # if both detect and record used, split the stream after the depay element + # to avoid encoding for recording + if use_detect and use_record: + pipeline = [*pipeline, f"tee name={DEPAYED_STREAM_NAME}", "queue"] + + if use_detect: + # decendants should override get_detect_decoder_pipeline to provide correct decoder element + detect_decoder_pipeline = self.get_detect_decoder_pipeline() + if detect_decoder_pipeline is None or len(detect_decoder_pipeline) == 0: + return self._build_gst_pipeline( + self._get_default_pipeline(), use_detect=True, use_record=False + ) + pipeline.extend(detect_decoder_pipeline) + + return self._build_gst_pipeline( + pipeline, use_detect=use_detect, use_record=use_record + ) + + def build(self, use_detect: bool, use_record: bool) -> List[str]: + if self.raw_pipeline is None or len(self.raw_pipeline) == 0: + full_pipeline = self._build(use_detect, use_record) + else: + full_pipeline = self.raw_pipeline + + pipeline_args = [ + f"{item} !".split(" ") for item in full_pipeline if len(item) > 0 + ] + pipeline_args = [item for sublist in pipeline_args for item in sublist] + return ["gst-launch-1.0", "-q", *pipeline_args][:-1] + + +class GstreamerNvidia(GstreamerBaseBuilder): + def __init__(self, width, height, name, format="I420") -> None: + super().__init__(width, height, name, format) + + @staticmethod + def accept(plugins: List[str]) -> bool: + """ + Accept method receives a list of plugins and return true if the builder can hande the current list + Builder should check all necessary pluguns before returning True + """ + required_plugins = ["nvv4l2decoder", "nvvidconv"] + for plugin in required_plugins: + if plugin not in plugins: + return False + return True + + def get_detect_decoder_pipeline(self) -> List[str]: + return [ + "nvv4l2decoder enable-max-performance=true", + "video/x-raw(memory:NVMM),format=NV12", + "nvvidconv", + f"video/x-raw,width=(int){self.width},height=(int){self.height},format=(string){self.format}", + ] + + +# A list of available builders. Please put on top more specific builders and keep the GstreamerBaseBuilder as a last builder +GSTREAMER_BUILDERS = [GstreamerNvidia, GstreamerBaseBuilder] + + +def get_gstreamer_builder(width, height, name, format="I420") -> GstreamerBaseBuilder: + available_plugins = gst_inspect_find_codec(codec=None) + for builder in GSTREAMER_BUILDERS: + if builder.accept(available_plugins): + return builder(width, height, name, format) + return diff --git a/frigate/http.py b/frigate/http.py index f21d1f8e6c..687cc08dd1 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -13,7 +13,6 @@ from pathlib import Path import cv2 -from flask.helpers import send_file import numpy as np from flask import ( @@ -26,10 +25,10 @@ request, ) -from peewee import SqliteDatabase, operator, fn, DoesNotExist, Value +from peewee import SqliteDatabase, operator, fn, DoesNotExist from playhouse.shortcuts import model_to_dict -from frigate.const import CLIPS_DIR, RECORD_DIR +from frigate.const import CLIPS_DIR from frigate.models import Event, Recordings from frigate.stats import stats_snapshot from frigate.util import calculate_region @@ -453,11 +452,11 @@ def events(): def config(): config = current_app.frigate_config.dict() - # add in the ffmpeg_cmds + # add in the decoder_cmds for camera_name, camera in current_app.frigate_config.cameras.items(): camera_dict = config["cameras"][camera_name] - camera_dict["ffmpeg_cmds"] = copy.deepcopy(camera.ffmpeg_cmds) - for cmd in camera_dict["ffmpeg_cmds"]: + camera_dict["decoder_cmds"] = copy.deepcopy(camera.decoder_cmds) + for cmd in camera_dict["decoder_cmds"]: cmd["cmd"] = " ".join(cmd["cmd"]) return jsonify(config) diff --git a/frigate/output.py b/frigate/output.py index dff8fc47e2..f368605873 100644 --- a/frigate/output.py +++ b/frigate/output.py @@ -9,6 +9,7 @@ import threading from multiprocessing import shared_memory from wsgiref.simple_server import make_server +from frigate.log import LogPipe import cv2 import numpy as np @@ -32,21 +33,28 @@ def __init__(self, in_width, in_height, out_width, out_height, quality): ffmpeg_cmd = f"ffmpeg -f rawvideo -pix_fmt yuv420p -video_size {in_width}x{in_height} -i pipe: -f mpegts -s {out_width}x{out_height} -codec:v mpeg1video -q {quality} -bf 0 pipe:".split( " " ) + self.logpipe = LogPipe( + "ffmpeg.converter", logging.ERROR) self.process = sp.Popen( ffmpeg_cmd, stdout=sp.PIPE, - stderr=sp.DEVNULL, + stderr=self.logpipe, stdin=sp.PIPE, start_new_session=True, ) def write(self, b): - self.process.stdin.write(b) + try: + self.process.stdin.write(b) + except Exception: + self.logpipe.dump() + return False def read(self, length): try: return self.process.stdout.read1(length) except ValueError: + self.logpipe.dump() return False def exit(self): diff --git a/frigate/record.py b/frigate/record.py index e184895c9c..8d5bc6d681 100644 --- a/frigate/record.py +++ b/frigate/record.py @@ -1,4 +1,5 @@ import datetime +from dateutil import tz import itertools import logging import multiprocessing as mp @@ -14,10 +15,14 @@ from pathlib import Path import psutil -from peewee import JOIN, DoesNotExist +from peewee import DoesNotExist from frigate.config import RetainModeEnum, FrigateConfig -from frigate.const import CACHE_DIR, RECORD_DIR +from frigate.const import ( + CACHE_DIR, + RECORD_DIR, + GSTREAMER_RECORD_SUFFIX, +) from frigate.models import Event, Recordings from frigate.util import area @@ -68,7 +73,7 @@ def move_files(self): files_in_use = [] for process in psutil.process_iter(): try: - if process.name() != "ffmpeg": + if process.name() not in ["ffmpeg", "gst-launch-1.0"]: continue flist = process.open_files() if flist: @@ -88,7 +93,14 @@ def move_files(self): cache_path = os.path.join(CACHE_DIR, f) basename = os.path.splitext(f)[0] camera, date = basename.rsplit("-", maxsplit=1) - start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S") + if camera.endswith(GSTREAMER_RECORD_SUFFIX): + camera = camera.split(GSTREAMER_RECORD_SUFFIX)[0] + creation_time = ( + os.path.getctime(cache_path) + ) + start_time = datetime.datetime.utcfromtimestamp(creation_time) + else: + start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S") grouped_recordings[camera].append( { diff --git a/frigate/test/conftest.py b/frigate/test/conftest.py new file mode 100644 index 0000000000..5b8986f627 --- /dev/null +++ b/frigate/test/conftest.py @@ -0,0 +1,26 @@ +import pytest +from unittest import mock +import sys + +def fake_open(filename, *args, **kvargs): + if filename == '/labelmap.txt': + content = "0 person\n1 bicycle" + else: + raise FileNotFoundError(filename) + file_object = mock.mock_open(read_data=content).return_value + file_object.__iter__.return_value = content.splitlines(True) + return file_object + +@pytest.fixture(scope="session", autouse=True) +def filesystem_mock(): + with mock.patch("builtins.open", new=fake_open, create=True): + yield + +# monkeypatch tflite_runtime +# in case of moving to the pytest completely, this can be done in more pyhonic way +module = type(sys)('tflite_runtime') +sys.modules['tflite_runtime'] = module + +module = type(sys)('tflite_runtime.interpreter') +module.load_delegate = mock.MagicMock() +sys.modules['tflite_runtime.interpreter'] = module diff --git a/frigate/test/requirements.test.txt b/frigate/test/requirements.test.txt new file mode 100644 index 0000000000..910e0ab0b9 --- /dev/null +++ b/frigate/test/requirements.test.txt @@ -0,0 +1,17 @@ +opencv-python-headless +numpy +imutils +scipy +psutil +Flask +paho-mqtt +PyYAML +matplotlib +click +setproctitle +peewee +peewee_migrate +pydantic +zeroconf +ws4py +pytest \ No newline at end of file diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index 0052c6f287..0c08069278 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -1,4 +1,5 @@ import unittest +from unittest import mock import numpy as np from pydantic import ValidationError from frigate.config import ( @@ -247,7 +248,7 @@ def test_default_input_args(self): assert config == frigate_config.dict(exclude_unset=True) runtime_config = frigate_config.runtime_config - assert "-rtsp_transport" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] + assert "-rtsp_transport" in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] def test_ffmpeg_params_global(self): config = { @@ -276,7 +277,7 @@ def test_ffmpeg_params_global(self): assert config == frigate_config.dict(exclude_unset=True) runtime_config = frigate_config.runtime_config - assert "-re" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] + assert "-re" in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] def test_ffmpeg_params_camera(self): config = { @@ -306,8 +307,8 @@ def test_ffmpeg_params_camera(self): assert config == frigate_config.dict(exclude_unset=True) runtime_config = frigate_config.runtime_config - assert "-re" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] - assert "test" not in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] + assert "-re" in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] + assert "test" not in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] def test_ffmpeg_params_input(self): config = { @@ -341,10 +342,10 @@ def test_ffmpeg_params_input(self): assert config == frigate_config.dict(exclude_unset=True) runtime_config = frigate_config.runtime_config - assert "-re" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] - assert "test" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] - assert "test2" not in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] - assert "test3" not in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] + assert "-re" in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] + assert "test" in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] + assert "test2" not in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] + assert "test3" not in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] def test_inherit_clips_retention(self): config = { @@ -512,9 +513,9 @@ def test_role_assigned_but_not_enabled(self): assert config == frigate_config.dict(exclude_unset=True) runtime_config = frigate_config.runtime_config - ffmpeg_cmds = runtime_config.cameras["back"].ffmpeg_cmds - assert len(ffmpeg_cmds) == 1 - assert not "clips" in ffmpeg_cmds[0]["roles"] + decoder_cmds = runtime_config.cameras["back"].decoder_cmds + assert len(decoder_cmds) == 1 + assert not "clips" in decoder_cmds[0]["roles"] def test_max_disappeared_default(self): config = { @@ -1267,6 +1268,115 @@ def test_fails_on_bad_camera_name(self): self.assertRaises( ValidationError, lambda: frigate_config.runtime_config.cameras ) + + @mock.patch( + "frigate.config.gst_discover", + return_value={"video": "video/x-h265"}, + ) + @mock.patch( + "frigate.gstreamer.gst_inspect_find_codec", + return_value=["nvv4l2decoder", "nvvidconv"], + ) + def test_gstreamer_params_camera_gstautodetect_detect( + self, mock_find_codec, mock_gst_discover + ): + config = { + "mqtt": {"host": "mqtt"}, + "rtmp": {"enabled": False}, + "cameras": { + "back": { + "gstreamer": { + "inputs": [ + { + "path": "rtsp://10.0.0.1:554/video", + "roles": ["detect"], + "input_options": ["protocols=tcp"], + } + ], + }, + "detect": { + "height": 1080, + "width": 1920, + "fps": 5, + }, + "objects": { + "track": ["person", "dog"], + "filters": {"dog": {"threshold": 0.7}}, + }, + } + }, + } + + frigate_config = FrigateConfig(**config) + assert config == frigate_config.dict(exclude_unset=True) + + runtime_config = frigate_config.runtime_config + + mock_find_codec.assert_called_with(codec=None) + mock_gst_discover.assert_called_with( + "rtsp://10.0.0.1:554/video", "back", ("width", "height", "video", "audio") + ) + + assert "nvv4l2decoder" in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] + assert ( + "video/x-raw,width=(int)1920,height=(int)1080,format=(string)I420" + in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] + ) + # custom rtspsrc arguments + assert "protocols=tcp" in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] + + @mock.patch( + "frigate.config.gst_discover", + side_effect=Exception("should not call gst_discover"), + ) + @mock.patch( + "frigate.gstreamer.gst_inspect_find_codec", + return_value=["nvv4l2decoder", "nvvidconv"], + ) + def test_gstreamer_params_camera_gstautodetect_detect( + self, mock_find_codec, mock_gst_discover + ): + config = { + "mqtt": {"host": "mqtt"}, + "rtmp": {"enabled": False}, + "cameras": { + "back": { + "gstreamer": { + "inputs": [ + { + "path": "rtsp://10.0.0.1:554/video", + "roles": ["detect"], + "video_format": "video/x-h265", + } + ], + }, + "detect": { + "height": 1080, + "width": 1920, + "fps": 5, + }, + "objects": { + "track": ["person", "dog"], + "filters": {"dog": {"threshold": 0.7}}, + }, + } + }, + } + frigate_config = FrigateConfig(**config) + assert config == frigate_config.dict(exclude_unset=True) + + runtime_config = frigate_config.runtime_config + + mock_find_codec.assert_called_with(codec=None) + mock_gst_discover.assert_not_called() + + assert "nvv4l2decoder" in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] + assert ( + "video/x-raw,width=(int)1920,height=(int)1080,format=(string)I420" + in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] + ) + # default rtspsrc arguments + assert "latency=0" in runtime_config.cameras["back"].decoder_cmds[0]["cmd"] if __name__ == "__main__": diff --git a/frigate/test/test_gstreamer.py b/frigate/test/test_gstreamer.py new file mode 100644 index 0000000000..63ceb06d61 --- /dev/null +++ b/frigate/test/test_gstreamer.py @@ -0,0 +1,522 @@ +from unittest import TestCase, main, mock +from typing import List +from frigate.gstreamer import ( + gst_discover, + gst_inspect_find_codec, + GstreamerBaseBuilder, + get_gstreamer_builder, +) + + +class TestGstTools(TestCase): + def test_gst_discover(self): + response = r""" + Topology: + unknown: application/x-rtp, media=(string)video, payload=(int)98, clock-rate=(int)90000, encoding-name=(string)H265, profile-id=(string)1, sprop-sps=(string)"QgEBAWAAAAMAsAAAAwAAAwBaoAeCAeFja5JMvTcBAQEAgA\=\=", sprop-pps=(string)"RAHA8vA8kA\=\=", sprop-vps=(string)"QAEMAf//AWAAAAMAsAAAAwAAAwBarAk\=", a-packetization-supported=(string)DH, a-rtppayload-supported=(string)DH, a-framerate=(string)15.000000, a-recvonly=(string)"", ssrc=(uint)1080610384, clock-base=(uint)52816, seqnum-base=(uint)52816, npt-start=(guint64)0, play-speed=(double)1, play-scale=(double)1 + video: video/x-h265, stream-format=(string)byte-stream, alignment=(string)au, width=(int)960, height=(int)480, chroma-format=(string)4:2:0, bit-depth-luma=(uint)8, bit-depth-chroma=(uint)8, parsed=(boolean)true, profile=(string)main, tier=(string)main, level=(string)3 + Tags: + video codec: H.265 (Main Profile) + + Codec: + video/x-h265, stream-format=(string)byte-stream, alignment=(string)au, width=(int)960, height=(int)480, chroma-format=(string)4:2:0, bit-depth-luma=(uint)8, bit-depth-chroma=(uint)8, parsed=(boolean)true, profile=(string)main, tier=(string)main, level=(string)3 + Additional info: + None + Stream ID: b9049c323800fa1dbf0c9c2f5d6dcf0e63b50fc2c5030d1c14e44a893d14e333/video:0:0:RTP:AVP:98 + Width: 960 + Height: 480 + Depth: 24 + Frame rate: 0/1 + audio: audio/x-alaw, channels=(int)1, rate=(int)8000 + Properties: + Duration: 99:99:99.999999999 + Seekable: no + Live: yes + Tags: + video codec: H.265 (Main Profile) + """ + with mock.patch( + "frigate.gstreamer.sp.check_output", return_value=response + ) as mock_checkout: + result = gst_discover( + "path to stream", + "cam1", + tuple(["width", "height", "video", "audio", "notinthelist"]), + ) + assert result == { + "height": "480", + "video": "video/x-h265", + "width": "960", + "audio": "audio/x-alaw", + } + mock_checkout.assert_called_once_with( + ["gst-discoverer-1.0", "-v", "path to stream"], + universal_newlines=True, + start_new_session=True, + stderr=None, + timeout=15, + ) + + def test_gst_inspect_find_codec(self): + response = """ + omx: omxh264dec: OpenMAX H.264 Video Decoder + omx: omxh264enc: OpenMAX H.264 Video Encoder + libav: avenc_h264_omx: libav OpenMAX IL H.264 video encoder encoder + libav: avdec_h264: libav H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10 decoder + typefindfunctions: video/x-h264: h264, x264, 264 + rtp: rtph264depay: RTP H264 depayloader + rtp: rtph264pay: RTP H264 payloader + nvvideo4linux2: nvv4l2h264enc: V4L2 H.264 Encoder + uvch264: uvch264mjpgdemux: UVC H264 MJPG Demuxer + uvch264: uvch264src: UVC H264 Source + videoparsersbad: h264parse: H.264 parser + omx: omxh265dec: OpenMAX H.265 Video Decoder + omx: omxh265enc: OpenMAX H.265 Video Encoder + libav: avdec_h265: libav HEVC (High Efficiency Video Coding) decoder + typefindfunctions: video/x-h265: h265, x265, 265 + rtp: rtph265depay: RTP H265 depayloader + rtp: rtph265pay: RTP H265 payloader + nvvideo4linux2: nvv4l2h265enc: V4L2 H.265 Encoder + videoparsersbad: h265parse: H.265 parser + """ + with mock.patch( + "frigate.gstreamer.sp.check_output", return_value=response + ) as mock_checkout: + result = gst_inspect_find_codec("h264") + assert result == [ + "omxh264dec", + "omxh264enc", + "avenc_h264_omx", + "avdec_h264", + "video/x-h264", + "rtph264depay", + "rtph264pay", + "nvv4l2h264enc", + "uvch264mjpgdemux", + "uvch264src", + "h264parse", + ] + mock_checkout.assert_called_once_with( + ["gst-inspect-1.0"], + universal_newlines=True, + start_new_session=True, + stderr=None, + ) + result = gst_inspect_find_codec("h265") + assert result == [ + "omxh265dec", + "omxh265enc", + "avdec_h265", + "video/x-h265", + "rtph265depay", + "rtph265pay", + "nvv4l2h265enc", + "h265parse", + ] + + +class TestGstreamerBaseBuilder(TestCase): + def setUp(self): + self.builder = GstreamerBaseBuilder(320, 240, "cam_name") + + def test_accept(self): + assert ( + GstreamerBaseBuilder.accept([]) == True + ), "GstreamerBaseBuilder should accept any plugin list" + + def test_build(self): + assert self.builder.build(use_detect=True, use_record=False) == [ + "gst-launch-1.0", + "-q", + "videotestsrc", + "pattern=19", + "!", + "video/x-raw,width=(int)320,height=(int)240,format=(string)I420,framerate=20/1", + "!", + "videorate", + "drop-only=true", + "!", + "video/x-raw,framerate=1/10", + "!", + "fdsink", + ] + + def test_with_source(self): + test_data = [ + ( + "rtsp://some/path1", + None, + [ + 'rtspsrc location="rtsp://some/path1" name=rtp_stream latency=0 do-timestamp=true' + ], + ), + ( + "rtsps://some/path1", + None, + [ + 'rtspsrc location="rtsps://some/path1" name=rtp_stream latency=0 do-timestamp=true' + ], + ), + ( + "rtsp://some/path2", + [], + [ + 'rtspsrc location="rtsp://some/path2" name=rtp_stream latency=0 do-timestamp=true' + ], + ), + ( + "rtsp://some/path3", + ["do-timestamp=true"], + [ + 'rtspsrc location="rtsp://some/path3" name=rtp_stream do-timestamp=true' + ], + ), + ( + "rtsp://some/path4", + ["do-timestamp=true", "! rtpjitterbuffer do-lost=true"], + [ + 'rtspsrc location="rtsp://some/path4" name=rtp_stream do-timestamp=true', + "rtpjitterbuffer do-lost=true", + ], + ), + ( + "rtsp://some/path4", + ["do-timestamp=true", "!", "rtpjitterbuffer", "do-lost=true"], + [ + 'rtspsrc location="rtsp://some/path4" name=rtp_stream do-timestamp=true', + "rtpjitterbuffer do-lost=true", + ], + ), + ( + "rtmp://some/path", + None, + ['rtmpsrc location="rtmp://some/path" name=rtp_stream'], + ), + ( + "rtmpt://some/path", + None, + ['rtmpsrc location="rtmpt://some/path" name=rtp_stream'], + ), + ( + "rtmps://some/path", + None, + ['rtmpsrc location="rtmps://some/path" name=rtp_stream'], + ), + ( + "rtmpe://some/path", + None, + ['rtmpsrc location="rtmpe://some/path" name=rtp_stream'], + ), + ( + "rtmfp://some/path", + None, + ['rtmpsrc location="rtmfp://some/path" name=rtp_stream'], + ), + ( + "myawesomesource key1=value1 ! myawesomeplugin key2=value2 option", + None, + ["myawesomesource key1=value1", "myawesomeplugin key2=value2 option"], + ), + ] + for url, options, expected in test_data: + with self.subTest(url=url, options=options): + assert self.builder.with_source(url, options).input_pipeline == expected + + +class TestGstreamerBuilderFactory(TestCase): + def build_detect_pipeline(self, builder: GstreamerBaseBuilder) -> List[str]: + return builder.with_source( + "rtsp://some/url", ["protocols=tcp", "latency=0", "do-timestamp=true"] + ).build(use_detect=True, use_record=False) + + @mock.patch("frigate.gstreamer.gst_inspect_find_codec", return_value=[]) + def test_find_codec_nothing(self, mock_find_codec): + """ + Since gst_inspect_find_codec return no plugins available, gstreamer_builder_factory should return + base GstreamerBaseBuilder, which creates a `videotestsrc` pipeline + """ + builder = get_gstreamer_builder(320, 240, "cam_name") + mock_find_codec.assert_called_with(codec=None) + assert self.build_detect_pipeline(builder) == [ + "gst-launch-1.0", + "-q", + "videotestsrc", + "pattern=19", + "!", + "video/x-raw,width=(int)320,height=(int)240,format=(string)I420,framerate=20/1", + "!", + "videorate", + "drop-only=true", + "!", + "video/x-raw,framerate=1/10", + "!", + "fdsink", + ] + + +class TestGstreamerNvidia(TestCase): + def build_detect_pipeline(self, builder: GstreamerBaseBuilder) -> List[str]: + return builder.with_source( + "rtsp://some/url", ["protocols=tcp", "latency=0", "do-timestamp=true"] + ).with_video_format("h264") + + @mock.patch( + "frigate.gstreamer.gst_inspect_find_codec", + return_value=["nvv4l2decoder", "nvvidconv"], + ) + def test_detect(self, mock_find_codec): + builder = get_gstreamer_builder(320, 240, "cam_name") + mock_find_codec.assert_called_with(codec=None) + assert self.build_detect_pipeline(builder).build( + use_detect=True, use_record=False + ) == [ + "gst-launch-1.0", + "-q", + "rtspsrc", + 'location="rtsp://some/url"', + "name=rtp_stream", + "protocols=tcp", + "latency=0", + "do-timestamp=true", + "!", + "rtph264depay", + "!", + "nvv4l2decoder", + "enable-max-performance=true", + "!", + "video/x-raw(memory:NVMM),format=NV12", + "!", + "nvvidconv", + "!", + "video/x-raw,width=(int)320,height=(int)240,format=(string)I420", + "!", + "fdsink", + ] + + @mock.patch( + "frigate.gstreamer.gst_inspect_find_codec", + return_value=["nvv4l2decoder", "nvvidconv"], + ) + def test_detect_record(self, mock_find_codec): + builder = get_gstreamer_builder(320, 240, "cam_name") + mock_find_codec.assert_called_with(codec=None) + assert self.build_detect_pipeline(builder).build( + use_detect=True, use_record=True + ) == [ + "gst-launch-1.0", + "-q", + "rtspsrc", + 'location="rtsp://some/url"', + "name=rtp_stream", + "protocols=tcp", + "latency=0", + "do-timestamp=true", + "!", + "rtpjitterbuffer", + "do-lost=true", + "drop-on-latency=true", + "!", + "rtph264depay", + "!", + "tee", + "name=depayed_stream", + "!", + "queue", + "!", + "nvv4l2decoder", + "enable-max-performance=true", + "!", + "video/x-raw(memory:NVMM),format=NV12", + "!", + "nvvidconv", + "!", + "video/x-raw,width=(int)320,height=(int)240,format=(string)I420", + "!", + "fdsink", + "depayed_stream.", + "!", + "queue", + "!", + "h264parse", + "!", + "splitmuxsink", + "async-finalize=true", + "send-keyframe-requests=true", + "max-size-bytes=0", + "location=/tmp/cache/cam_name-gstsplitmuxchunk-%05d.mp4", + "max-size-time=10000000000", + ] + + @mock.patch( + "frigate.gstreamer.gst_inspect_find_codec", + return_value=["nvv4l2decoder", "nvvidconv"], + ) + def test_record_only(self, mock_find_codec): + builder = get_gstreamer_builder(320, 240, "cam_name") + mock_find_codec.assert_called_with(codec=None) + assert self.build_detect_pipeline(builder).build( + use_detect=False, use_record=True + ) == [ + "gst-launch-1.0", + "-q", + "rtspsrc", + 'location="rtsp://some/url"', + "name=rtp_stream", + "protocols=tcp", + "latency=0", + "do-timestamp=true", + "!", + "rtpjitterbuffer", + "do-lost=true", + "drop-on-latency=true", + "!", + "rtph264depay", + "!", + "queue", + "!", + "h264parse", + "!", + "splitmuxsink", + "async-finalize=true", + "send-keyframe-requests=true", + "max-size-bytes=0", + "location=/tmp/cache/cam_name-gstsplitmuxchunk-%05d.mp4", + "max-size-time=10000000000", + ] + + @mock.patch( + "frigate.gstreamer.gst_inspect_find_codec", + return_value=["nvv4l2decoder", "nvvidconv"], + ) + def test_detect_record_audio(self, mock_find_codec): + builder = get_gstreamer_builder(320, 240, "cam_name") + mock_find_codec.assert_called_with(codec=None) + assert self.build_detect_pipeline(builder).with_video_format( + "video/x-h265" + ).with_audio_pipeline( + ["rtppcmadepay", "alawdec", "audioconvert", "queue", "avenc_aac"] + ).build( + use_detect=True, use_record=True + ) == [ + "gst-launch-1.0", + "-q", + "rtspsrc", + 'location="rtsp://some/url"', + "name=rtp_stream", + "protocols=tcp", + "latency=0", + "do-timestamp=true", + "!", + "rtpjitterbuffer", + "do-lost=true", + "drop-on-latency=true", + "!", + "rtph265depay", + "!", + "tee", + "name=depayed_stream", + "!", + "queue", + "!", + "nvv4l2decoder", + "enable-max-performance=true", + "!", + "video/x-raw(memory:NVMM),format=NV12", + "!", + "nvvidconv", + "!", + "video/x-raw,width=(int)320,height=(int)240,format=(string)I420", + "!", + "fdsink", + "depayed_stream.", + "!", + "queue", + "!", + "h265parse", + "!", + "splitmuxsink", + "async-finalize=true", + "send-keyframe-requests=true", + "max-size-bytes=0", + "name=mux", + "muxer=mp4mux", + "location=/tmp/cache/cam_name-gstsplitmuxchunk-%05d.mp4", + "max-size-time=10000000000", + "rtp_stream.", + "!", + "queue", + "!", + "rtppcmadepay", + "!", + "alawdec", + "!", + "audioconvert", + "!", + "queue", + "!", + "avenc_aac", + "!", + "mux.audio_0", + ] + + @mock.patch( + "frigate.gstreamer.gst_inspect_find_codec", + return_value=["nvv4l2decoder", "nvvidconv"], + ) + def test_detect_record_audio_by_format(self, mock_find_codec): + builder = get_gstreamer_builder(320, 240, "cam_name") + mock_find_codec.assert_called_with(codec=None) + assert self.build_detect_pipeline(builder).with_audio_format( + "audio/mpeg" + ).build(use_detect=False, use_record=True) == [ + "gst-launch-1.0", + "-q", + "rtspsrc", + 'location="rtsp://some/url"', + "name=rtp_stream", + "protocols=tcp", + "latency=0", + "do-timestamp=true", + "!", + "rtpjitterbuffer", + "do-lost=true", + "drop-on-latency=true", + "!", + "rtph264depay", + "!", + "queue", + "!", + "h264parse", + "!", + "splitmuxsink", + "async-finalize=true", + "send-keyframe-requests=true", + "max-size-bytes=0", + "name=mux", + "muxer=mp4mux", + "location=/tmp/cache/cam_name-gstsplitmuxchunk-%05d.mp4", + "max-size-time=10000000000", + "rtp_stream.", + "!", + "queue", + "!", + "rtpmp4gdepay", + "!", + "aacparse", + "!", + "mux.audio_0", + ] + + @mock.patch( + "frigate.gstreamer.gst_inspect_find_codec", + return_value=[], + ) + def test_raw_pipeline(self, mock_find_codec): + builder = get_gstreamer_builder(320, 240, "cam_name") + mock_find_codec.assert_called_with(codec=None) + assert builder.with_raw_pipeline(["videotestsrc", "autovideosink"]).build( + use_detect=True, use_record=True + ) == ["gst-launch-1.0", "-q", "videotestsrc", "!", "autovideosink"] + + +if __name__ == "__main__": + main(verbosity=2) diff --git a/frigate/util.py b/frigate/util.py index f11c0b0f9b..2dd1b5569e 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -12,7 +12,7 @@ import traceback from abc import ABC, abstractmethod from multiprocessing import shared_memory -from typing import AnyStr +from typing import AnyStr, Dict, List, Optional import cv2 import matplotlib.pyplot as plt @@ -34,7 +34,7 @@ def deep_merge(dct1: dict, dct2: dict, override=False, merge_lists=False) -> dic for k, v2 in dct2.items(): if k in merged: v1 = merged[k] - if isinstance(v1, dict) and isinstance(v2, collections.Mapping): + if isinstance(v1, dict) and isinstance(v2, collections.abc.Mapping): merged[k] = deep_merge(v1, v2, override) elif isinstance(v1, list) and isinstance(v2, list): if merge_lists: @@ -533,6 +533,13 @@ def clipped(obj, frame_shape): else: return False +def empty_or_none(obj) -> bool: + if obj is None: + return True + if len(obj) == 0: + return True + return False + def restart_frigate(): proc = psutil.Process(1) diff --git a/frigate/video.py b/frigate/video.py index e38f206aa7..cd818785e1 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -16,7 +16,7 @@ from setproctitle import setproctitle from frigate.config import CameraConfig, DetectConfig -from frigate.edgetpu import RemoteObjectDetector +from frigate.detection import RemoteObjectDetector from frigate.log import LogPipe from frigate.motion import MotionDetector from frigate.objects import ObjectTracker @@ -78,38 +78,38 @@ def filtered(obj, objects_to_track, object_filters): def create_tensor_input(frame, model_shape, region): cropped_frame = yuv_region_2_rgb(frame, region) - # Resize to 300x300 if needed - if cropped_frame.shape != (model_shape[0], model_shape[1], 3): + # Resize to the model_shape if needed + height, width = model_shape + if cropped_frame.shape != (height, width, 3): cropped_frame = cv2.resize( - cropped_frame, dsize=model_shape, interpolation=cv2.INTER_LINEAR + cropped_frame, dsize=(width, height), interpolation=cv2.INTER_LINEAR ) + # Return a tensor of shape: [height, width, 3] in RGB format + return cropped_frame - # Expand dimensions since the model expects images to have shape: [1, height, width, 3] - return np.expand_dims(cropped_frame, axis=0) - -def stop_ffmpeg(ffmpeg_process, logger): - logger.info("Terminating the existing ffmpeg process...") - ffmpeg_process.terminate() +def stop_decoder(decoder_process, logger): + logger.info("Terminating the existing decoder process...") + decoder_process.terminate() try: - logger.info("Waiting for ffmpeg to exit gracefully...") - ffmpeg_process.communicate(timeout=30) + logger.info("Waiting for decoder to exit gracefully...") + decoder_process.communicate(timeout=30) except sp.TimeoutExpired: - logger.info("FFmpeg didnt exit. Force killing...") - ffmpeg_process.kill() - ffmpeg_process.communicate() - ffmpeg_process = None + logger.info("decoder didnt exit. Force killing...") + decoder_process.kill() + decoder_process.communicate() + decoder_process = None -def start_or_restart_ffmpeg( - ffmpeg_cmd, logger, logpipe: LogPipe, frame_size=None, ffmpeg_process=None +def start_or_restart_decoder( + decoder_cmd, logger, logpipe: LogPipe, frame_size=None, decoder_process=None ): - if ffmpeg_process is not None: - stop_ffmpeg(ffmpeg_process, logger) + if decoder_process is not None: + stop_decoder(decoder_process, logger) if frame_size is None: process = sp.Popen( - ffmpeg_cmd, + decoder_cmd, stdout=sp.DEVNULL, stderr=logpipe, stdin=sp.DEVNULL, @@ -117,7 +117,7 @@ def start_or_restart_ffmpeg( ) else: process = sp.Popen( - ffmpeg_cmd, + decoder_cmd, stdout=sp.PIPE, stderr=logpipe, stdin=sp.DEVNULL, @@ -128,7 +128,7 @@ def start_or_restart_ffmpeg( def capture_frames( - ffmpeg_process, + decoder_process, camera_name, frame_shape, frame_manager: FrameManager, @@ -151,13 +151,13 @@ def capture_frames( frame_name = f"{camera_name}{current_frame.value}" frame_buffer = frame_manager.create(frame_name, frame_size) try: - frame_buffer[:] = ffmpeg_process.stdout.read(frame_size) + frame_buffer[:] = decoder_process.stdout.read(frame_size) except Exception as e: - logger.error(f"{camera_name}: Unable to read frames from ffmpeg process.") + logger.error(f"{camera_name}: Unable to read frames from decoder process.") - if ffmpeg_process.poll() != None: + if decoder_process.poll() != None: logger.error( - f"{camera_name}: ffmpeg process is not running. exiting capture thread..." + f"{camera_name}: decoder process is not running. exiting capture thread..." ) frame_manager.delete(frame_name) break @@ -180,38 +180,38 @@ def capture_frames( class CameraWatchdog(threading.Thread): def __init__( - self, camera_name, config, frame_queue, camera_fps, ffmpeg_pid, stop_event + self, camera_name, config, frame_queue, camera_fps, decoder_pid, stop_event ): threading.Thread.__init__(self) self.logger = logging.getLogger(f"watchdog.{camera_name}") self.camera_name = camera_name self.config = config self.capture_thread = None - self.ffmpeg_detect_process = None - self.logpipe = LogPipe(f"ffmpeg.{self.camera_name}.detect", logging.ERROR) - self.ffmpeg_other_processes = [] + self.decoder_detect_process = None + self.logpipe = LogPipe(f"decoder.{self.camera_name}.detect", logging.ERROR) + self.decoder_other_processes = [] self.camera_fps = camera_fps - self.ffmpeg_pid = ffmpeg_pid + self.decoder_pid = decoder_pid self.frame_queue = frame_queue self.frame_shape = self.config.frame_shape_yuv self.frame_size = self.frame_shape[0] * self.frame_shape[1] self.stop_event = stop_event def run(self): - self.start_ffmpeg_detect() + self.start_decoder_detect() - for c in self.config.ffmpeg_cmds: + for c in self.config.decoder_cmds: if "detect" in c["roles"]: continue logpipe = LogPipe( - f"ffmpeg.{self.camera_name}.{'_'.join(sorted(c['roles']))}", + f"decoder.{self.camera_name}.{'_'.join(sorted(c['roles']))}", logging.ERROR, ) - self.ffmpeg_other_processes.append( + self.decoder_other_processes.append( { "cmd": c["cmd"], "logpipe": logpipe, - "process": start_or_restart_ffmpeg(c["cmd"], self.logger, logpipe), + "process": start_or_restart_decoder(c["cmd"], self.logger, logpipe), } ) @@ -221,52 +221,52 @@ def run(self): if not self.capture_thread.is_alive(): self.logger.error( - f"Ffmpeg process crashed unexpectedly for {self.camera_name}." + f"decoder process crashed unexpectedly for {self.camera_name}." ) self.logger.error( - "The following ffmpeg logs include the last 100 lines prior to exit." + "The following decoder logs include the last 100 lines prior to exit." ) self.logpipe.dump() - self.start_ffmpeg_detect() + self.start_decoder_detect() elif now - self.capture_thread.current_frame.value > 20: self.logger.info( - f"No frames received from {self.camera_name} in 20 seconds. Exiting ffmpeg..." + f"No frames received from {self.camera_name} in 20 seconds. Exiting decoder..." ) - self.ffmpeg_detect_process.terminate() + self.decoder_detect_process.terminate() try: - self.logger.info("Waiting for ffmpeg to exit gracefully...") - self.ffmpeg_detect_process.communicate(timeout=30) + self.logger.info("Waiting for decoder to exit gracefully...") + self.decoder_detect_process.communicate(timeout=30) except sp.TimeoutExpired: - self.logger.info("FFmpeg didnt exit. Force killing...") - self.ffmpeg_detect_process.kill() - self.ffmpeg_detect_process.communicate() + self.logger.info("decoder didnt exit. Force killing...") + self.decoder_detect_process.kill() + self.decoder_detect_process.communicate() - for p in self.ffmpeg_other_processes: + for p in self.decoder_other_processes: poll = p["process"].poll() if poll is None: continue p["logpipe"].dump() - p["process"] = start_or_restart_ffmpeg( - p["cmd"], self.logger, p["logpipe"], ffmpeg_process=p["process"] + p["process"] = start_or_restart_decoder( + p["cmd"], self.logger, p["logpipe"], decoder_process=p["process"] ) - stop_ffmpeg(self.ffmpeg_detect_process, self.logger) - for p in self.ffmpeg_other_processes: - stop_ffmpeg(p["process"], self.logger) + stop_decoder(self.decoder_detect_process, self.logger) + for p in self.decoder_other_processes: + stop_decoder(p["process"], self.logger) p["logpipe"].close() self.logpipe.close() - def start_ffmpeg_detect(self): - ffmpeg_cmd = [ - c["cmd"] for c in self.config.ffmpeg_cmds if "detect" in c["roles"] + def start_decoder_detect(self): + decoder_cmd = [ + c["cmd"] for c in self.config.decoder_cmds if "detect" in c["roles"] ][0] - self.ffmpeg_detect_process = start_or_restart_ffmpeg( - ffmpeg_cmd, self.logger, self.logpipe, self.frame_size + self.decoder_detect_process = start_or_restart_decoder( + decoder_cmd, self.logger, self.logpipe, self.frame_size ) - self.ffmpeg_pid.value = self.ffmpeg_detect_process.pid + self.decoder_pid.value = self.decoder_detect_process.pid self.capture_thread = CameraCapture( self.camera_name, - self.ffmpeg_detect_process, + self.decoder_detect_process, self.frame_shape, self.frame_queue, self.camera_fps, @@ -275,7 +275,7 @@ def start_ffmpeg_detect(self): class CameraCapture(threading.Thread): - def __init__(self, camera_name, ffmpeg_process, frame_shape, frame_queue, fps): + def __init__(self, camera_name, decoder_process, frame_shape, frame_queue, fps): threading.Thread.__init__(self) self.name = f"capture:{camera_name}" self.camera_name = camera_name @@ -284,14 +284,14 @@ def __init__(self, camera_name, ffmpeg_process, frame_shape, frame_queue, fps): self.fps = fps self.skipped_fps = EventsPerSecond() self.frame_manager = SharedMemoryFrameManager() - self.ffmpeg_process = ffmpeg_process + self.decoder_process = decoder_process self.current_frame = mp.Value("d", 0.0) self.last_frame = 0 def run(self): self.skipped_fps.start() capture_frames( - self.ffmpeg_process, + self.decoder_process, self.camera_name, self.frame_shape, self.frame_manager, @@ -317,7 +317,7 @@ def receiveSignal(signalNumber, frame): config, frame_queue, process_info["camera_fps"], - process_info["ffmpeg_pid"], + process_info["decoder_pid"], stop_event, ) camera_watchdog.start() diff --git a/frigate/watchdog.py b/frigate/watchdog.py index 73cf86240a..9c8c7c34d1 100644 --- a/frigate/watchdog.py +++ b/frigate/watchdog.py @@ -27,7 +27,7 @@ def run(self): # check the detection processes for detector in self.detectors.values(): detection_start = detector.detection_start.value - if detection_start > 0.0 and now - detection_start > 10: + if detection_start > 0.0 and now - detection_start > 30: logger.info( "Detection appears to be stuck. Restarting detection process..." ) diff --git a/process_clip.py b/process_clip.py index 5699985264..f6f046820a 100644 --- a/process_clip.py +++ b/process_clip.py @@ -67,8 +67,8 @@ def __init__(self, clip_path, frame_shape, config: FrigateConfig): self.config = config self.camera_config = self.config.cameras["camera"] self.frame_shape = self.camera_config.frame_shape - self.ffmpeg_cmd = [ - c["cmd"] for c in self.camera_config.ffmpeg_cmds if "detect" in c["roles"] + self.decoder_cmd = [ + c["cmd"] for c in self.camera_config.decoder_cmds if "detect" in c["roles"] ][0] self.frame_manager = SharedMemoryFrameManager() self.frame_queue = mp.Queue() @@ -84,7 +84,7 @@ def load_frames(self): * self.camera_config.frame_shape_yuv[1] ) ffmpeg_process = start_or_restart_ffmpeg( - self.ffmpeg_cmd, logger, sp.DEVNULL, frame_size + self.decoder_cmd, logger, sp.DEVNULL, frame_size ) capture_frames( ffmpeg_process,