search

blakeblackshear / frigate

NVR with realtime local object detection for IP cameras
https://frigate.video
MIT License
19.06k stars 1.74k forks source link

[Support]: Errors after being on for a few hours #7225

Closed tv21 closed 1 year ago

tv21 commented 1 year ago

Describe the problem you are having

Log starts producing errors after being on for a couple hours, but I can't seem to figure out what is causing them.

Version

0.12.1-367d724

Frigate config file

mqtt:
  enabled: True
  host: 192.168.1.100
  port: 1883
  topic_prefix: frigate
  client_id: frigate
  user: username
  password: password
  stats_interval: 60

snapshots:
  enabled: True
  bounding_box: True

detect:
  width: 896
  height: 672
  fps: 30

detectors:
  coral:
    type: edgetpu
    device: usb

record:
  enabled: True
  events:
    pre_capture: 10
    post_capture: 10
    retain:
      default: 10
      mode: all

ffmpeg:
  hwaccel_args: preset-vaapi
  output_args:
   record: preset-record-generic-audio-copy

go2rtc:
  streams:
    your_reolink_camera:
      - "ffmpeg:http://192.168.1.51/flv?port=1935&app=bcs&stream=channel0_main.bcs&user=user&password=password"
      - "ffmpeg:your_reolink_camera#audio=opus"
    your_reolink_camera_sub:
      - "ffmpeg:http://192.168.1.51/flv?port=1935&app=bcs&stream=channel0_ext.bcs&user=user&password=password"

    your_reolink2_camera:
      - "ffmpeg:http://192.168.1.52/flv?port=1935&app=bcs&stream=channel0_main.bcs&user=user&password=password"
      - "ffmpeg:your_reolink_camera#audio=opus"
    your_reolink2_camera_sub:
      - "ffmpeg:http://192.168.1.52/flv?port=1935&app=bcs&stream=channel0_ext.bcs&user=user&password=password"

    your_reolink3_camera:
      - "ffmpeg:http://192.168.1.53/flv?port=1935&app=bcs&stream=channel0_main.bcs&user=user&password=password"
      - "ffmpeg:your_reolink_camera#audio=opus"
    your_reolink3_camera_sub:
      - "ffmpeg:http://192.168.1.53/flv?port=1935&app=bcs&stream=channel0_ext.bcs&user=user&password=password"

    your_reolink4_camera:
      - "ffmpeg:http://192.168.1.54/flv?port=1935&app=bcs&stream=channel0_main.bcs&user=user&password=password"
      - "ffmpeg:your_reolink_camera#audio=opus"
    your_reolink4_camera_sub:
      - "ffmpeg:http://192.168.1.54/flv?port=1935&app=bcs&stream=channel0_ext.bcs&user=user&password=password"

cameras:
  Driveway:
    ffmpeg:
      hwaccel_args: preset-vaapi
      inputs:
        - path: rtsp://127.0.0.1:8554/your_reolink_camera?video=copy&audio=aac
          input_args: preset-rtsp-restream
          roles:
            - record
        - path: rtsp://127.0.0.1:8554/your_reolink_camera_sub?video=copy
          input_args: preset-rtsp-restream
          roles:
            - detect
    detect:
      enabled: True
    objects:
      track:
        - person
        - dog
        - cat
        - bicycle
        - bird
        - car
        - horse
        - sports ball
        - bear
        - motorcycle
      filters:
        person:
          threshold: 0.8
        car:
          threshold: 0.8
          mask:
            - 0,363,222,315,204,192,0,221
            - 544,209,646,204,644,153,542,156
        bird:
          threshold: 0.8
        cat:
          threshold: 0.8

  Deck:
    ffmpeg:
      hwaccel_args: preset-vaapi
      inputs:
        - path: rtsp://127.0.0.1:8554/your_reolink2_camera?video=copy&audio=aac
          input_args: preset-rtsp-restream
          roles:
            - record
        - path: rtsp://127.0.0.1:8554/your_reolink2_camera_sub?video=copy
          input_args: preset-rtsp-restream
          roles:
            - detect
    detect:
      enabled: True
    objects:
      track:
        - person
        - dog
        - cat
        - bicycle
        - bird
        - car
        - horse
        - sports ball
        - bear
        - motorcycle
      filters:
        person:
          threshold: 0.8
        car:
          threshold: 0.8
        bird:
          threshold: 0.8
        cat:
          threshold: 0.8

  Front_Yard:
    ffmpeg:
      hwaccel_args: preset-vaapi
      inputs:
        - path: rtsp://127.0.0.1:8554/your_reolink3_camera?video=copy&audio=aac
          input_args: preset-rtsp-restream
          roles:
            - record
        - path: rtsp://127.0.0.1:8554/your_reolink3_camera_sub?video=copy
          input_args: preset-rtsp-restream
          roles:
            - detect
    detect:
      enabled: True
    objects:
      track:
        - person
        - dog
        - cat
        - bicycle
        - bird
        - car
        - horse
        - sports ball
        - bear
        - motorcycle
      filters:
        person:
          threshold: 0.8
        car:
          threshold: 0.8
        bird:
          threshold: 0.8
        cat:
          threshold: 0.8

  Back_Yard:
    ffmpeg:
      hwaccel_args: preset-vaapi
      inputs:
        - path: rtsp://127.0.0.1:8554/your_reolink4_camera?video=copy&audio=aac
          input_args: preset-rtsp-restream
          roles:
            - record
        - path: rtsp://127.0.0.1:8554/your_reolink4_camera_sub?video=copy
          input_args: preset-rtsp-restream
          roles:
            - detect
    detect:
      enabled: True
    objects:
      track:
        - person
        - dog
        - cat
        - bicycle
        - bird
        - car
        - horse
        - sports ball
        - bear
        - motorcycle
      filters:
        person:
          threshold: 0.8
        car:
          threshold: 0.8
        bird:
          threshold: 0.8
        cat:
          threshold: 0.8

rtmp:
  enabled: False
live:
  height: 1080
  quality: 1

Relevant log output

2023-07-18 16:57:04.280901065  [2023-07-18 16:57:04] frigate.record                 ERROR   : Error occurred when attempting to maintain recording cache
2023-07-18 16:57:04.280904511  [2023-07-18 16:57:04] frigate.record                 ERROR   : list index out of range
2023-07-18 16:57:09.220426170  [2023-07-18 16:57:09] frigate.record                 ERROR   : Error occurred when attempting to maintain recording cache
2023-07-18 16:57:09.220679922  [2023-07-18 16:57:09] frigate.record                 ERROR   : list index out of range
2023-07-18 16:57:14.362622200  [2023-07-18 16:57:14] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Back_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:14.365126845  [2023-07-18 16:57:14] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Deck. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:14.367312986  [2023-07-18 16:57:14] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Driveway. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:14.370288701  [2023-07-18 16:57:14] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Front_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:14.377192594  [2023-07-18 16:57:14] frigate.record                 ERROR   : Error occurred when attempting to maintain recording cache
2023-07-18 16:57:14.377465645  [2023-07-18 16:57:14] frigate.record                 ERROR   : list index out of range
2023-07-18 16:57:19.233826029  [2023-07-18 16:57:19] frigate.record                 ERROR   : Error occurred when attempting to maintain recording cache
2023-07-18 16:57:19.234214741  [2023-07-18 16:57:19] frigate.record                 ERROR   : list index out of range
2023-07-18 16:57:24.328827102  [2023-07-18 16:57:24] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Back_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:24.329693609  [2023-07-18 16:57:24] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Deck. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:24.330715789  [2023-07-18 16:57:24] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Driveway. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:24.331780189  [2023-07-18 16:57:24] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Front_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:24.333974243  [2023-07-18 16:57:24] frigate.record                 ERROR   : Error occurred when attempting to maintain recording cache
2023-07-18 16:57:24.334023980  [2023-07-18 16:57:24] frigate.record                 ERROR   : list index out of range
2023-07-18 16:57:34.418037909  [2023-07-18 16:57:34] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Back_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:34.419044347  [2023-07-18 16:57:34] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Deck. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:34.421137259  [2023-07-18 16:57:34] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Driveway. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:34.422624565  [2023-07-18 16:57:34] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Front_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:44.612962043  [2023-07-18 16:57:44] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Back_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:44.615547964  [2023-07-18 16:57:44] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Deck. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:44.618554115  [2023-07-18 16:57:44] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Driveway. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 16:57:44.621485401  [2023-07-18 16:57:44] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Front_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:24.238839295  [2023-07-18 17:00:24] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Back_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:24.240622379  [2023-07-18 17:00:24] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Deck. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:24.243318065  [2023-07-18 17:00:24] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Driveway. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:24.245318185  [2023-07-18 17:00:24] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Front_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:34.255208862  [2023-07-18 17:00:34] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Back_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:34.262683203  [2023-07-18 17:00:34] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Deck. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:34.271558070  [2023-07-18 17:00:34] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Driveway. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:34.276489935  [2023-07-18 17:00:34] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Front_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:44.626030112  [2023-07-18 17:00:44] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Back_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:44.630830949  [2023-07-18 17:00:44] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Deck. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:44.632317893  [2023-07-18 17:00:44] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Driveway. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:00:44.636094623  [2023-07-18 17:00:44] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Front_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:01:04.211889154  [2023-07-18 17:01:04] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Back_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:01:04.213283439  [2023-07-18 17:01:04] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Deck. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:01:04.215379350  [2023-07-18 17:01:04] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Front_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:01:14.263580371  [2023-07-18 17:01:14] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Back_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:01:14.265514764  [2023-07-18 17:01:14] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Deck. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:01:14.267344231  [2023-07-18 17:01:14] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Driveway. Keeping the 5 most recent segments out of 6 and discarding the rest...
2023-07-18 17:01:14.269189466  [2023-07-18 17:01:14] frigate.record                 WARNING : Unable to keep up with recording segments in cache for Front_Yard. Keeping the 5 most recent segments out of 6 and discarding the rest...

FFprobe output from your camera

[{"return_code":0,"stderr":"","stdout":{"programs":[],"streams":[{"avg_frame_rate":"0/0","codec_long_name":"H.264/AVC/MPEG-4AVC/MPEG-4part10","height":1920,"width":2560},{"avg_frame_rate":"0/0","codec_long_name":"AAC(AdvancedAudioCoding)"}]}},{"return_code":0,"stderr":"","stdout":{"programs":[],"streams":[{"avg_frame_rate":"0/0","codec_long_name":"H.264/AVC/MPEG-4AVC/MPEG-4part10","height":672,"width":896}]}}]

Frigate stats

{"Back_Yard":{"camera_fps":30.1,"capture_pid":309,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":325,"pid":299,"process_fps":30.1,"skipped_fps":0.0},"Deck":{"camera_fps":30.1,"capture_pid":301,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":308,"pid":297,"process_fps":30.0,"skipped_fps":0.0},"Driveway":{"camera_fps":30.1,"capture_pid":300,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":306,"pid":296,"process_fps":30.1,"skipped_fps":0.0},"Front_Yard":{"camera_fps":30.1,"capture_pid":307,"detection_enabled":1,"detection_fps":0.0,"ffmpeg_pid":314,"pid":298,"process_fps":30.0,"skipped_fps":0.0},"cpu_usages":{"%Cpu(s):":{"cpu":"id,","mem":"2.0"},"1":{"cpu":"0.0","mem":"0.0"},"101":{"cpu":"0.0","mem":"0.0"},"112":{"cpu":"14.6","mem":"6.7"},"118":{"cpu":"0.0","mem":"0.2"},"131":{"cpu":"0.0","mem":"0.4"},"132":{"cpu":"0.0","mem":"0.1"},"133":{"cpu":"0.0","mem":"0.5"},"134":{"cpu":"0.0","mem":"0.1"},"15":{"cpu":"0.0","mem":"0.0"},"16":{"cpu":"0.0","mem":"0.0"},"16932":{"cpu":"0.0","mem":"0.0"},"16954":{"cpu":"0.3","mem":"0.0"},"24":{"cpu":"0.0","mem":"0.0"},"25":{"cpu":"0.0","mem":"0.0"},"26":{"cpu":"0.0","mem":"0.0"},"27":{"cpu":"0.0","mem":"0.0"},"28":{"cpu":"0.0","mem":"0.0"},"281":{"cpu":"0.0","mem":"1.0"},"287":{"cpu":"3.0","mem":"0.1"},"288":{"cpu":"0.0","mem":"1.4"},"29":{"cpu":"0.0","mem":"0.0"},"290":{"cpu":"11.3","mem":"1.2"},"296":{"cpu":"6.6","mem":"1.2"},"297":{"cpu":"6.3","mem":"1.3"},"298":{"cpu":"6.6","mem":"1.3"},"299":{"cpu":"6.6","mem":"1.3"},"30":{"cpu":"0.0","mem":"0.0"},"300":{"cpu":"14.2","mem":"1.1"},"301":{"cpu":"14.9","mem":"1.1"},"306":{"cpu":"18.2","mem":"0.7"},"307":{"cpu":"14.6","mem":"1.1"},"308":{"cpu":"17.9","mem":"0.7"},"309":{"cpu":"13.6","mem":"1.1"},"31":{"cpu":"0.0","mem":"0.0"},"314":{"cpu":"18.9","mem":"0.7"},"317":{"cpu":"3.0","mem":"0.3"},"318":{"cpu":"1.7","mem":"0.3"},"32":{"cpu":"0.0","mem":"0.0"},"320":{"cpu":"1.3","mem":"0.3"},"323":{"cpu":"2.3","mem":"0.3"},"325":{"cpu":"17.5","mem":"0.7"},"326":{"cpu":"0.0","mem":"0.8"},"329":{"cpu":"3.0","mem":"0.4"},"331":{"cpu":"3.0","mem":"0.3"},"335":{"cpu":"0.0","mem":"0.2"},"336":{"cpu":"3.3","mem":"0.4"},"337":{"cpu":"2.0","mem":"0.5"},"339":{"cpu":"1.7","mem":"0.3"},"340":{"cpu":"0.0","mem":"0.2"},"341":{"cpu":"1.3","mem":"0.3"},"343":{"cpu":"3.3","mem":"0.5"},"345":{"cpu":"0.0","mem":"0.2"},"348":{"cpu":"3.6","mem":"0.4"},"349":{"cpu":"0.0","mem":"0.2"},"41":{"cpu":"0.0","mem":"0.0"},"42":{"cpu":"0.0","mem":"0.0"},"79":{"cpu":"0.0","mem":"0.0"},"81":{"cpu":"0.0","mem":"0.0"},"82":{"cpu":"0.0","mem":"0.0"},"89":{"cpu":"13.9","mem":"0.2"},"MiB":{"cpu":"5896.8","mem":"avail"},"PID":{"cpu":"%CPU","mem":"%MEM"},"Tasks:":{"cpu":"stopped,","mem":"0"},"top":{"cpu":"average:","mem":"2.08,"}},"detection_fps":0.0,"detectors":{"coral":{"detection_start":0.0,"inference_speed":13.04,"pid":288}},"gpu_usages":{"intel-vaapi":{"gpu":"7.92 %","mem":"- %"}},"service":{"last_updated":1689720629,"latest_version":"0.12.1","storage":{"/dev/shm":{"free":2141.6,"mount_type":"tmpfs","total":2147.5,"used":5.9},"/media/frigate/clips":{"free":921626.8,"mount_type":"ext4","total":982240.0,"used":10642.7},"/media/frigate/recordings":{"free":967901.2,"mount_type":"ext4","total":983351.1,"used":5431.1},"/tmp/cache":{"free":939.6,"mount_type":"tmpfs","total":1000.0,"used":60.4}},"temperatures":{},"uptime":28703,"version":"0.12.1-367d724"}}

Operating system

Debian

Install method

Docker Compose

Coral version

USB

Network connection

Wired

Camera make and model

reolink RLC-510A

Any other information that may be helpful

OS is really Ubuntu 22.04 Server, but that was not an option.

Note I am posting this for a family member, this will be a duplicate of https://github.com/blakeblackshear/frigate/issues/7214 if GitHub ever allows that issue to be viewed.

NickM-27 commented 1 year ago

I see some things wrong here:

  1. You should absolutely not be running detect at 30 fps, there is no reason to do that. The detect fps value does not need to match the camera
  2. Enable debug logs for recording and report back once the errors start occurring again (if they do after fixing number 1)
logger:
  logs:
    frigate.record: debug
tv21 commented 1 year ago 
1. You should absolutely not be running detect at 30 fps, there is no reason to do that. The detect fps value does not need to match the camera

He asked me to ask you, "What is the recommended fps on the detect then? Would something like 5 be normal?" He also said he had only set that to match the camera because he had read somewhere it was supposed to, but apparently that was wrong.

NickM-27 commented 1 year ago

5 is the default and sufficient for most cases

tv21 commented 1 year ago

Lowering detect to 5 appears to have fixed the problem for him, thank you for your assistance!