search

blakeblackshear / frigate

NVR with realtime local object detection for IP cameras
https://frigate.video
MIT License
18.75k stars 1.7k forks source link

[Support]: Object Detection not working after switching to TensorRT #5579

Closed austindetzel closed 1 year ago

austindetzel commented 1 year ago

Describe the problem you are having

I have switched to tensorrt from cpu detection about 2 days ago. I am not getting any object detection now. I ahve recently needed to fix other problems as well including cache size and shm size, as well as birdseye settings. I luckily have frigate connected to home assistant which provides a fair amount of historical information. It seems like motion is working as seen here:

image

But object detection seems to be stuck in time since when I switched to tensorrt as seen here:

image

I have tried modifying different thresholds with no changes, at the moment everything is almost back to default settings as you will see in the config. I am not sure what I did wrong here. As suggested I waited a while to let log files form, however after about an hour and a half they are still fairly slim. Something I couldn't have said a few days ago but I bet makes troubleshooting hard now.

Version

0.12.0-27A31E7

Frigate config file

# yaml-language-server: $schema=http://192.168.40.10:5000/api/config/schema.json

mqtt:
  host: 192.168.40.10
  port: 1883

#detectors:
#  cpu1:
#    type: cpu
#    num_threads: 4

detectors:
  tensorrt:
    type: tensorrt
    device: 0 #This is the default, select the first GPU

model:
  path: /trt-models/yolov4-tiny-288.trt
  input_tensor: nchw
  input_pixel_format: rgb
  width: 416
  height: 416

motion:
  threshold: 50

record:
    enabled: True
    retain:
        days: 14
        mode: all
    events:
        retain:
            default: 28
            mode: motion 

detect:
  enabled: True

objects:
  track:
    - person
    - car
    - cat
    - dog
    - bird

go2rtc:
  #log:
  #  level: "trace"
  streams:
    Front_Door: 
      - rtsp://admin:[redacted]@192.168.40.22/cam/realmonitor?channel=1&subtype=0
      - "ffmpeg:Front_Door#audio=opus"
    Front_Yard:
      - rtsp://admin:[redacted]@192.168.40.23/cam/realmonitor?channel=1&subtype=0
      - "ffmpeg:Front_Yard#audio=opus"
    Shed_Camera:
      - rtsp://admin:[redacted]@192.168.40.21/cam/realmonitor?channel=1&subtype=0
      - "ffmpeg:Shed_Camera#audio=opus"
    Driveway_Doorbell:
      - rtsp://admin:[redacted]@192.168.40.24/cam/realmonitor?channel=1&subtype=0
      - "ffmpeg:Driveway_Doorbell#audio=opus"
    Living_Room:
      - rtsp://admin:[redacted]@192.168.40.20/live
      - "ffmpeg:Front_Door#audio=opus"
    Barn:
      - http://192.168.40.25/flv?port=1935&app=bcs&stream=channel0_main.bcs&user=admin&password=[redacted]
      - "ffmpeg:Barn#audio=opus"
    Septic_Shed:
      - http://192.168.40.26/flv?port=1935&app=bcs&stream=channel0_main.bcs&user=admin&password=[redacted]
      - "ffmpeg:Septic_Shed#audio=opus"
    Septic_Driveway:
      - http://192.168.40.28/flv?port=1935&app=bcs&stream=channel0_main.bcs&user=admin&password=[redacted]
      - "ffmpeg:Septic_Driveway#audio=opus"
    Septic_Inside:
      - http://192.168.40.27/flv?port=1935&app=bcs&stream=channel0_main.bcs&user=admin&password=[redacted]
      - "ffmpeg:Septic_Inside#audio=opus"

ffmpeg:
  global_args: -hide_banner -loglevel warning
  hwaccel_args: preset-nvidia-h264
  output_args:
    record: preset-record-generic-audio-copy

cameras:
    Front_Door:
        ffmpeg:
            inputs:                
                - path:  rtsp://127.0.0.1:8554/Front_Door
                  input_args: preset-rtsp-restream
                  roles:
                    - record
                    - detect
        detect:
            width: 1920 
            height: 1080 

    Front_Yard:
        ffmpeg:
            inputs:
                - path: rtsp://127.0.0.1:8554/Front_Yard
                  input_args: preset-rtsp-restream
                  roles:
                    - detect
                    - record
                  hwaccel_args: preset-nvidia-h265
        detect:
            width: 1920
            height: 1080

    Shed_Camera:
        ffmpeg:    
            inputs:
                - path: rtsp://127.0.0.1:8554/Shed_Camera
                  input_args: preset-rtsp-restream-low-latency
                  roles:
                    - detect
                    - record
        detect:
            width: 3840 
            height: 2160 

    Driveway_Doorbell:
        ffmpeg:     
            inputs:
                - path: rtsp://127.0.0.1:8554/Driveway_Doorbell 
                  input_args: preset-rtsp-restream
                  roles:
                    - detect
                    - record
        detect:
            width: 1920 
            height: 1080       

    Living_Room:
        ffmpeg:
            inputs:
                - path: rtsp://127.0.0.1:8554/Living_Room
                  input_args: preset-rtsp-restream
                  roles:
                    - detect
                    - record
        detect:
            #enabled: False
            width: 1920 
            height: 1080 
        birdseye:
          enabled: False

    Barn:
        ffmpeg:
            inputs:
                - path: rtsp://127.0.0.1:8554/Barn
                  input_args: preset-rtsp-restream-low-latency
                  roles:
                    - detect
                    - record
        detect:
            width: 2560
            height: 1920
        birdseye:
          mode: continuous

    Septic_Shed:
        ffmpeg:
            inputs:
                - path: rtsp://127.0.0.1:8554/Septic_Shed
                  input_args: preset-rtsp-restream-low-latency
                  roles:
                    - detect
                    - record
        detect:
            width: 2560 
            height: 1920 

    Septic_Driveway:
        ffmpeg:      
            inputs:
                - path: rtsp://127.0.0.1:8554/Septic_Driveway
                  input_args: preset-rtsp-restream
                  roles:
                    - detect
                    - record
        detect:
            width: 2560 
            height: 1440 

    Septic_Inside:
        ffmpeg:
            inputs:
                - path: rtsp://127.0.0.1:8554/Septic_Inside
                  input_args: preset-rtsp-restream-low-latency
                  roles:
                    - detect
                    - record
        detect:
            width: 2560 
            height: 1920 

birdseye:
  enabled: True
  restream: True
  width: 1080 #1280
  height: 720 #720
  quality: 25

snapshots:
  enabled: False
  clean_copy: False

Relevant log output

2023-02-23 18:10:19.263341007  [2023-02-23 18:10:19] frigate.app                    INFO    : Starting Frigate (0.12.0-27a31e7)
2023-02-23 18:10:19.292887845  [2023-02-23 18:10:19] peewee_migrate                 INFO    : Starting migrations
2023-02-23 18:10:19.326871339  [2023-02-23 18:10:19] peewee_migrate                 INFO    : There is nothing to migrate
2023-02-23 18:10:19.341440899  [2023-02-23 18:10:19] detector.tensorrt              INFO    : Starting detection process: 980
2023-02-23 18:10:19.352220207  [2023-02-23 18:10:19] frigate.app                    INFO    : Output process started: 982
2023-02-23 18:10:19.352732530  [2023-02-23 18:10:19] frigate.app                    INFO    : Camera processor started for Front_Door: 986
2023-02-23 18:10:19.352792883  [2023-02-23 18:10:19] frigate.app                    INFO    : Camera processor started for Front_Yard: 987
2023-02-23 18:10:19.357996800  [2023-02-23 18:10:19] frigate.app                    INFO    : Camera processor started for Shed_Camera: 991
2023-02-23 18:10:19.362446943  [2023-02-23 18:10:19] frigate.app                    INFO    : Camera processor started for Driveway_Doorbell: 992
2023-02-23 18:10:19.367259005  [2023-02-23 18:10:19] frigate.app                    INFO    : Camera processor started for Living_Room: 994
2023-02-23 18:10:19.370818458  [2023-02-23 18:10:19] frigate.app                    INFO    : Camera processor started for Barn: 995
2023-02-23 18:10:19.376387884  [2023-02-23 18:10:19] frigate.app                    INFO    : Camera processor started for Septic_Shed: 997
2023-02-23 18:10:19.380107259  [2023-02-23 18:10:19] frigate.app                    INFO    : Camera processor started for Septic_Driveway: 998
2023-02-23 18:10:19.384066445  [2023-02-23 18:10:19] frigate.app                    INFO    : Camera processor started for Septic_Inside: 1000
2023-02-23 18:10:19.387580523  [2023-02-23 18:10:19] frigate.app                    INFO    : Capture process started for Front_Door: 1003
2023-02-23 18:10:19.391938361  [2023-02-23 18:10:19] frigate.app                    INFO    : Capture process started for Front_Yard: 1006
2023-02-23 18:10:19.396560138  [2023-02-23 18:10:19] frigate.app                    INFO    : Capture process started for Shed_Camera: 1011
2023-02-23 18:10:19.403029742  [2023-02-23 18:10:19] frigate.app                    INFO    : Capture process started for Driveway_Doorbell: 1017
2023-02-23 18:10:19.408386984  [2023-02-23 18:10:19] frigate.app                    INFO    : Capture process started for Living_Room: 1023
2023-02-23 18:10:19.413486304  [2023-02-23 18:10:19] frigate.app                    INFO    : Capture process started for Barn: 1028
2023-02-23 18:10:19.419914754  [2023-02-23 18:10:19] frigate.app                    INFO    : Capture process started for Septic_Shed: 1034
2023-02-23 18:10:19.426373686  [2023-02-23 18:10:19] frigate.app                    INFO    : Capture process started for Septic_Driveway: 1040
2023-02-23 18:10:19.432466826  [2023-02-23 18:10:19] frigate.app                    INFO    : Capture process started for Septic_Inside: 1044
2023-02-23 18:10:19.642305979  [2023-02-23 18:10:19] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] Init CUDA: CPU +321, GPU +0, now: CPU 408, GPU 284 (MiB)
2023-02-23 18:10:19.647814040  [2023-02-23 18:10:19] frigate.detectors.plugins.tensorrt INFO    : Loaded engine size: 12 MiB
2023-02-23 18:10:20.162049560  [2023-02-23 18:10:20] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +855, GPU +358, now: CPU 1298, GPU 658 (MiB)
2023-02-23 18:10:20.244360102  [2023-02-23 18:10:20] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] Init cuDNN: CPU +126, GPU +58, now: CPU 1424, GPU 716 (MiB)
2023-02-23 18:10:20.245274829  [2023-02-23 18:10:20] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +11, now: CPU 0, GPU 11 (MiB)
2023-02-23 18:10:20.247670400  [2023-02-23 18:10:20] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 1411, GPU 708 (MiB)
2023-02-23 18:10:20.247719137  [2023-02-23 18:10:20] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 1411, GPU 716 (MiB)
2023-02-23 18:10:20.247740309  [2023-02-23 18:10:20] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +3, now: CPU 0, GPU 14 (MiB)
2023-02-23 18:11:27.927567131  [2023-02-23 18:11:27] frigate.record                 WARNING : Unable to find file from recordings database: /media/frigate/recordings/2023-02-03/06/Driveway_Doorbell/05.41.mp4

FFprobe output from your camera

"[\n  {\n    \"return_code\": 0,\n    \"stderr\": {},\n    \"stdout\": {\n      \"programs\": [],\n      \"streams\": [\n        {\n          \"avg_frame_rate\": \"15/1\",\n          \"codec_long_name\": \"H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10\",\n          \"height\": 1080,\n          \"width\": 1920\n        },\n        {\n          \"avg_frame_rate\": \"0/0\",\n          \"codec_long_name\": \"AAC (Advanced Audio Coding)\"\n        }\n      ]\n    }\n  }\n]"

Frigate stats

{"Barn":{"camera_fps":5.1,"capture_pid":1028,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":1033,"pid":995,"process_fps":4.8,"skipped_fps":0.0},"Driveway_Doorbell":{"camera_fps":5.0,"capture_pid":1017,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":1022,"pid":992,"process_fps":4.6,"skipped_fps":0.0},"Front_Door":{"camera_fps":5.0,"capture_pid":1003,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":1007,"pid":986,"process_fps":4.6,"skipped_fps":0.0},"Front_Yard":{"camera_fps":5.0,"capture_pid":1006,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":1012,"pid":987,"process_fps":4.6,"skipped_fps":0.0},"Living_Room":{"camera_fps":5.1,"capture_pid":1023,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":1029,"pid":994,"process_fps":4.6,"skipped_fps":0.0},"Septic_Driveway":{"camera_fps":5.0,"capture_pid":1040,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":1060,"pid":998,"process_fps":4.6,"skipped_fps":0.0},"Septic_Inside":{"camera_fps":5.0,"capture_pid":1044,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":1061,"pid":1000,"process_fps":4.6,"skipped_fps":0.0},"Septic_Shed":{"camera_fps":5.1,"capture_pid":1034,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":1038,"pid":997,"process_fps":4.7,"skipped_fps":0.0},"Shed_Camera":{"camera_fps":5.1,"capture_pid":1011,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":1018,"pid":991,"process_fps":4.6,"skipped_fps":0.0},"cpu_usages":{"%Cpu(s):":{"cpu":"id,","mem":"10.0"},"1":{"cpu":"0.0","mem":"0.0"},"1000":{"cpu":"0.3","mem":"0.3"},"1002":{"cpu":"0.0","mem":"0.0"},"1003":{"cpu":"1.7","mem":"0.3"},"1006":{"cpu":"1.7","mem":"0.3"},"1007":{"cpu":"5.0","mem":"0.4"},"1010":{"cpu":"0.0","mem":"0.0"},"1011":{"cpu":"5.3","mem":"0.4"},"1012":{"cpu":"3.3","mem":"0.4"},"1017":{"cpu":"1.3","mem":"0.3"},"1018":{"cpu":"12.3","mem":"0.8"},"1019":{"cpu":"0.0","mem":"0.0"},"1022":{"cpu":"4.7","mem":"0.4"},"1023":{"cpu":"1.7","mem":"0.3"},"1027":{"cpu":"0.0","mem":"0.0"},"1028":{"cpu":"4.0","mem":"0.3"},"1029":{"cpu":"3.3","mem":"0.4"},"1033":{"cpu":"8.0","mem":"0.6"},"1034":{"cpu":"4.3","mem":"0.3"},"1038":{"cpu":"8.3","mem":"0.6"},"1039":{"cpu":"2.3","mem":"0.1"},"1040":{"cpu":"3.0","mem":"0.3"},"1044":{"cpu":"4.0","mem":"0.3"},"105":{"cpu":"0.0","mem":"0.0"},"1060":{"cpu":"6.3","mem":"0.5"},"1061":{"cpu":"7.7","mem":"0.6"},"128":{"cpu":"0.0","mem":"0.0"},"129":{"cpu":"0.0","mem":"0.0"},"130":{"cpu":"0.0","mem":"0.0"},"131":{"cpu":"0.0","mem":"0.0"},"132":{"cpu":"0.0","mem":"0.0"},"141":{"cpu":"0.0","mem":"0.0"},"15":{"cpu":"0.0","mem":"0.0"},"16":{"cpu":"0.0","mem":"0.0"},"162":{"cpu":"0.0","mem":"0.0"},"207":{"cpu":"0.0","mem":"0.0"},"24":{"cpu":"0.0","mem":"0.0"},"25":{"cpu":"0.0","mem":"0.0"},"26":{"cpu":"0.0","mem":"0.0"},"260":{"cpu":"0.0","mem":"0.0"},"27":{"cpu":"0.0","mem":"0.0"},"28":{"cpu":"0.0","mem":"0.0"},"29":{"cpu":"0.0","mem":"0.0"},"294":{"cpu":"0.0","mem":"0.0"},"30":{"cpu":"0.0","mem":"0.0"},"31":{"cpu":"0.0","mem":"0.0"},"324":{"cpu":"0.0","mem":"0.0"},"363":{"cpu":"0.0","mem":"0.0"},"40":{"cpu":"0.0","mem":"0.0"},"41":{"cpu":"0.0","mem":"0.0"},"4163":{"cpu":"0.0","mem":"0.0"},"421":{"cpu":"0.0","mem":"0.0"},"446":{"cpu":"0.0","mem":"0.0"},"473":{"cpu":"0.0","mem":"0.0"},"495":{"cpu":"0.0","mem":"0.0"},"521":{"cpu":"0.0","mem":"0.0"},"564":{"cpu":"0.0","mem":"0.0"},"615":{"cpu":"0.0","mem":"0.0"},"641":{"cpu":"0.0","mem":"0.0"},"664":{"cpu":"0.0","mem":"0.0"},"681":{"cpu":"0.0","mem":"0.0"},"702":{"cpu":"0.0","mem":"0.0"},"731":{"cpu":"0.0","mem":"0.0"},"79":{"cpu":"0.0","mem":"0.0"},"80":{"cpu":"0.0","mem":"0.0"},"81":{"cpu":"0.0","mem":"0.0"},"88":{"cpu":"5.3","mem":"0.0"},"93":{"cpu":"2.0","mem":"1.1"},"973":{"cpu":"0.0","mem":"0.3"},"979":{"cpu":"0.3","mem":"0.0"},"980":{"cpu":"0.0","mem":"3.0"},"982":{"cpu":"12.7","mem":"0.3"},"985":{"cpu":"0.0","mem":"0.0"},"986":{"cpu":"0.3","mem":"0.3"},"987":{"cpu":"0.3","mem":"0.3"},"990":{"cpu":"0.0","mem":"0.0"},"991":{"cpu":"0.7","mem":"0.4"},"992":{"cpu":"0.3","mem":"0.3"},"993":{"cpu":"0.0","mem":"0.0"},"994":{"cpu":"0.3","mem":"0.3"},"995":{"cpu":"0.3","mem":"0.3"},"996":{"cpu":"0.0","mem":"0.0"},"997":{"cpu":"0.3","mem":"0.3"},"998":{"cpu":"0.7","mem":"0.3"},"999":{"cpu":"0.0","mem":"0.0"},"MiB":{"cpu":"31181.3","mem":"avail"},"PID":{"cpu":"%CPU","mem":"%MEM"},"Tasks:":{"cpu":"stopped,","mem":"0"},"top":{"cpu":"users,","mem":"load"}},"detection_fps":0.0,"detectors":{"tensorrt":{"detection_start":0.0,"inference_speed":1.78,"pid":980}},"gpu_usages":{"NVIDIA GeForce RTX 3060 Ti":{"gpu":"3 %","mem":"44.4 %"}},"service":{"last_updated":1677194390,"latest_version":"0.11.1","storage":{"/dev/shm":{"free":1013.9,"mount_type":"tmpfs","total":1073.7,"used":59.8},"/media/frigate/clips":{"free":1063103.6,"mount_type":"fuse.shfs","total":8637489.4,"used":7574385.8},"/media/frigate/recordings":{"free":1063103.6,"mount_type":"fuse.shfs","total":8637489.4,"used":7574385.8},"/tmp/cache":{"free":929.9,"mount_type":"tmpfs","total":1000.0,"used":70.1}},"temperatures":{},"uptime":571,"version":"0.12.0-27a31e7"}}

Operating system

UNRAID

Install method

Docker Compose

Coral version

Other

Network connection

Wired

Camera make and model

Amcrest, Reolink

Any other information that may be helpful

NVIDIA SMI Snapshot

image

NickM-27 commented 1 year ago

You are using the 288 model, but you have set the model size to 416 so this is incompatible. The model size needs to be correct for the model that is being used