[Config Support]: tmpfs filling up regardless of the allocated space

Describe the problem you are having

Since I started using Frigate (3-4 months ago), I have had the problem of tmpfs filling up. I know that many similar issues have been created, but in none of them I found a solution to this problem.

I tried with different configurations and allocated memory, even about 16GB, but the problem appeared sooner or later. Whenever I check the resources used, while the frigate is still running, everything is normal and only a few MB of memory is used, until one day it is 100% used.

I believe that this problem is critical, it causes all cameras to stop recording until a manual restart is performed, and I am afraid that such a situation will occur in the time frame when recordings will be particularly needed.

I suspect that this problem is related to the official Frigate addon for home-assistant. When it was turned off for a few days I did not experience this problem, but I may be wrong because the experiment time was too short and the frequency of the problem is non-deterministic.

Regardless of the cause of the problem, I believe that steps should be taken to ensure that the frigate is protected from such problems so that it can continue to function as an NVR.

Thank you in advance for your help.

Version

0.13.0-614a36a

Frigate config file

detectors:
  coral:
    type: edgetpu
    device: pci
mqtt:
  host: 10.10.10.100
  port: 1883
  user: addons
  password: xxx
ffmpeg:
  hwaccel_args: preset-vaapi
  output_args:
    record: preset-record-generic-audio-aac
objects:
  track:
  - person
  - face
  - dog
  - cat
  - car
  - package
  filters:
    dog:
      min_score: .7
      threshold: .8
    cat:
      min_score: .65
      threshold: .85
    face:
      min_score: .7
    package:
      min_score: .65
      threshold: .85
    license_plate:
      min_score: .6
    person:
      min_score: .80
      threshold: .80
    car:
      min_score: .65
      threshold: .85
cameras:
  gate:
    zones:
      wjazd:
        coordinates: 0,720,1141,720,1223,503,1259,316,1045,248
        objects:
        - person
        - cat
        - dog
        - car
    motion:
      mask:
      - 1280,146,1233,427,1144,720,1280,720
    objects:
      filters:
        car:
          mask: 1280,0,1280,314,1100,263,837,165,859,0
        person:
          mask:
          - 1259,316,1052,256,1057,150,974,184,839,167,880,0,1280,0,1280,296
          - 1062,153,1082,250,906,376,715,515,589,444,567,325,564,296
    ffmpeg:
      input_args: -avoid_negative_ts make_zero -fflags nobuffer+genpts+discardcorrupt
        -flags low_delay -strict experimental -rtsp_transport tcp -timeout 5000000
        -tag:v hvc1
      output_args:
        record: -f segment -segment_time 10 -segment_format mp4 -reset_timestamps
          1 -strftime 1 -c:v copy -tag:v hvc1 -bsf:v hevc_mp4toannexb -c:a aac
        rtmp: -c:v copy -c:a aac -f flv
      inputs:
      - path: rtsp://viewer:xxx@camera-gate.local:554/ch1/main/av_stream
        roles:
        - detect
        - record
        - rtmp
    rtmp:
      enabled: false # <-- RTMP should be disabled if your stream is not H264
    detect:
      width: 1280
      height: 720
      fps: 6
  front:
    zones:
      frontyard:
        coordinates: 1135,892,1568,842,1920,754,1920,1080,0,1080,0,827,189,859,564,895
        objects:
        - person
        - dog
        - car
        - cat
    objects:
      filters:
        car:
          mask:
            - 558,678,1171,664,1686,624,1920,581,1920,0,0,0,0,669
    ffmpeg:
      input_args: -avoid_negative_ts make_zero -fflags nobuffer+genpts+discardcorrupt
        -flags low_delay -strict experimental -rtsp_transport tcp -timeout 5000000
        -tag:v hvc1
      output_args:
        record: -f segment -segment_time 10 -segment_format mp4 -reset_timestamps
          1 -strftime 1 -c:v copy -tag:v hvc1 -bsf:v hevc_mp4toannexb -c:a aac
        rtmp: -c:v copy -c:a aac -f flv
      inputs:
      - path: rtsp://viewer:xxx@camera-front.local:554/ch1/main/av_stream
        roles:
        - record
        - detect
    rtmp:
      enabled: false # <-- RTMP should be disabled if your stream is not H264
    detect:
      width: 1920
      height: 1080
      fps: 6
  backyard:
    objects:
      filters:
        car:
          mask:
          - 1280,0,532,0,540,293,924,352,1103,720,1280,720
          - 617,555,629,313,806,321,788,564
          - 0,0,0,436,306,300,540,219,569,0,281,0
          - 284,518,571,549,632,446,278,433
      track:
      - person
      - dog
      - car
      - cat
    motion:
      mask:
        - 0,0,358,0,833,0,1280,0,1280,720,1171,720,1082,647,1055,595,1075,550,1027,525,1018,474,1103,483,1126,479,1126,468,1115,450,1103,430,1068,383,1166,389,1172,343,1163,313,1198,293,1202,262,1214,241,1234,207,1240,132,1169,70,1121,57,989,108,894,121,836,143,813,165,766,175,718,176,679,162,665,220,628,189,599,195,570,199,550,204,491,185,398,205,338,219,255,221,59,216,0,162
    ffmpeg:
      input_args: -avoid_negative_ts make_zero -fflags nobuffer+genpts+discardcorrupt
        -flags low_delay -strict experimental -rtsp_transport tcp -timeout 5000000
        -tag:v hvc1
      output_args:
        record: -f segment -segment_time 10 -segment_format mp4 -reset_timestamps
          1 -strftime 1 -c:v copy -tag:v hvc1 -bsf:v hevc_mp4toannexb -c:a aac
        rtmp: -c:v copy -c:a aac -f flv
      inputs:
      - path: rtsp://viewer:xxx@camera-backyard.local:554/ch1/main/av_stream
        roles:
        - detect
        - record
        - rtmp
    rtmp:
      enabled: false # <-- RTMP should be disabled if your stream is not H264
    detect:
      width: 1280
      height: 720
      fps: 6
record:
  enabled: true
  retain:
    days: 22
    mode: all
go2rtc:
  streams:
    gate:
    - ffmpeg:rtsp://viewer:xxx@camera-gate.local:554/ch1/main/av_stream#raw=-vf
      "fps=15,scale=-1:1440"#video=h264#audio=aac
    front:
    - ffmpeg:rtsp://viewer:xxx@camera-front.local:554/ch1/main/av_stream#raw=-vf
      "fps=15,scale=-1:1440"#video=h264#audio=aac
    backyard:
    - ffmpeg:rtsp://viewer:xxx@camera-backyard.local:554/ch1/main/av_stream#raw=-vf
      "fps=15,scale=-1:1440"#video=h264#audio=aac
  webrtc:
    candidates:
      - frigate.local:8555
      - stun:8555
snapshots:
  enabled: true
  timestamp: true
  bounding_box: true
  crop: true
  required_zones: []
  retain:
    default: 25
birdseye:
  enabled: true
  mode: continuous
  quality: 10
  width: 3840
  height: 2160
  restream: false
model:
  path: plus://xxx

Relevant log output

2024-01-29 21:37:37.424159911  [2024-01-29 21:37:37] ffmpeg.gate.detect             ERROR   : Error initializing output stream 0:1 -- 
2024-01-29 21:37:37.429914778  [2024-01-29 21:37:37] watchdog.backyard              ERROR   : Ffmpeg process crashed unexpectedly for backyard.
2024-01-29 21:37:37.429919193  [2024-01-29 21:37:37] watchdog.backyard              ERROR   : The following ffmpeg logs include the last 100 lines prior to exit.
2024-01-29 21:37:37.429921476  [2024-01-29 21:37:37] ffmpeg.backyard.detect         ERROR   : Could not write header for output file #0 (incorrect codec parameters ?): No space left on device
2024-01-29 21:37:37.430044122  [2024-01-29 21:37:37] ffmpeg.backyard.detect         ERROR   : Error initializing output stream 0:1 -- 
2024-01-29 21:37:39.050888464  [2024-01-29 21:37:39] frigate.video                  ERROR   : gate: Unable to read frames from ffmpeg process.
2024-01-29 21:37:39.051222140  [2024-01-29 21:37:39] frigate.video                  ERROR   : gate: ffmpeg process is not running. exiting capture thread...
2024-01-29 21:37:39.096590826  [2024-01-29 21:37:39] frigate.video                  ERROR   : front: Unable to read frames from ffmpeg process.
2024-01-29 21:37:39.096719211  [2024-01-29 21:37:39] frigate.video                  ERROR   : front: ffmpeg process is not running. exiting capture thread...
2024-01-29 21:37:39.140241073  [2024-01-29 21:37:39] frigate.video                  ERROR   : backyard: Unable to read frames from ffmpeg process.
2024-01-29 21:37:39.140337736  [2024-01-29 21:37:39] frigate.video                  ERROR   : backyard: ffmpeg process is not running. exiting capture thread...
2024-01-29 21:37:47.430141685  [2024-01-29 21:37:47] watchdog.gate                  ERROR   : Ffmpeg process crashed unexpectedly for gate.
2024-01-29 21:37:47.430217203  [2024-01-29 21:37:47] watchdog.gate                  ERROR   : The following ffmpeg logs include the last 100 lines prior to exit.
2024-01-29 21:37:47.430261986  [2024-01-29 21:37:47] ffmpeg.gate.detect             ERROR   : Could not write header for output file #0 (incorrect codec parameters ?): No space left on device
2024-01-29 21:37:47.430311000  [2024-01-29 21:37:47] ffmpeg.gate.detect             ERROR   : Error initializing output stream 0:1 -- 
2024-01-29 21:37:47.431316770  [2024-01-29 21:37:47] watchdog.front                 ERROR   : Ffmpeg process crashed unexpectedly for front.
2024-01-29 21:37:47.431423465  [2024-01-29 21:37:47] watchdog.front                 ERROR   : The following ffmpeg logs include the last 100 lines prior to exit.
2024-01-29 21:37:47.431494147  [2024-01-29 21:37:47] ffmpeg.front.detect            ERROR   : Could not write header for output file #0 (incorrect codec parameters ?): No space left on device
2024-01-29 21:37:47.431569680  [2024-01-29 21:37:47] ffmpeg.front.detect            ERROR   : Error initializing output stream 0:1 --
[...] (The same things keeps repeating itself)

# GO2RTC LOGS #
2024-01-26 02:14:28.735959207  [INFO] Preparing new go2rtc config...
2024-01-26 02:14:29.061328537  [INFO] Not injecting WebRTC candidates into go2rtc config as it has been set manually
2024-01-26 02:14:29.104032672  [INFO] Starting go2rtc...
2024-01-26 02:14:29.164030978  02:14:29.163 INF go2rtc version 1.8.4 linux/amd64
2024-01-26 02:14:29.164444341  02:14:29.164 INF [rtsp] listen addr=:8554
2024-01-26 02:14:29.164666740  02:14:29.164 INF [api] listen addr=:1984
2024-01-26 02:14:29.164787991  02:14:29.164 INF [webrtc] listen addr=:8555
2024-01-26 02:14:38.696162601  [INFO] Starting go2rtc healthcheck service...

Frigate stats

{
  "cameras": {
    "backyard": {
      "audio_dBFS": 0.0,
      "audio_rms": 0.0,
      "camera_fps": 0.0,
      "capture_pid": 354,
      "detection_enabled": 1,
      "detection_fps": 1.6,
      "ffmpeg_pid": 955636,
      "pid": 343,
      "process_fps": 6.2,
      "skipped_fps": 0.0
    },
    "front": {
      "audio_dBFS": 0.0,
      "audio_rms": 0.0,
      "camera_fps": 0.0,
      "capture_pid": 348,
      "detection_enabled": 1,
      "detection_fps": 8.1,
      "ffmpeg_pid": 955635,
      "pid": 342,
      "process_fps": 6.2,
      "skipped_fps": 0.0
    },
    "gate": {
      "audio_dBFS": 0.0,
      "audio_rms": 0.0,
      "camera_fps": 0.0,
      "capture_pid": 346,
      "detection_enabled": 1,
      "detection_fps": 2.3,
      "ffmpeg_pid": 955634,
      "pid": 341,
      "process_fps": 6.3,
      "skipped_fps": 0.0
    }
  },
  "cpu_usages": {
    "1": {
      "cmdline": "/package/admin/s6/command/s6-svscan -d4 -- /run/service",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "16": {
      "cmdline": "s6-supervise s6-linux-init-shutdownd",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "19": {
      "cmdline": "/package/admin/s6-linux-init/command/s6-linux-init-shutdownd -c /run/s6/basedir -g 3000 -C -B",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "25": {
      "cmdline": "s6-supervise go2rtc-healthcheck",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "26": {
      "cmdline": "s6-supervise nginx",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "27": {
      "cmdline": "s6-supervise nginx-log",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "28": {
      "cmdline": "s6-supervise s6rc-fdholder",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "29": {
      "cmdline": "s6-supervise go2rtc",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "30": {
      "cmdline": "s6-supervise frigate-log",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "31": {
      "cmdline": "s6-supervise frigate",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "32": {
      "cmdline": "s6-supervise s6rc-oneshot-runner",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "33": {
      "cmdline": "s6-supervise go2rtc-log",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "42": {
      "cmdline": "/package/admin/s6/command/s6-ipcserverd -1 -- /package/admin/s6/command/s6-ipcserver-access -v0 -E -l0 -i data/rules -- /package/admin/s6/command/s6-sudod -t 30000 -- /package/admin/s6-rc/command/s6-rc-oneshot-run -l ../.. --",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "43": {
      "cmdline": "/package/admin/s6-2.11.3.2/command/s6-fdholderd -1 -i data/rules",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "79": {
      "cmdline": "s6-log -b -- T 1 n0 s10000000 T /dev/shm/logs/nginx",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "82": {
      "cmdline": "s6-log -b -- T 1 n0 s10000000 T /dev/shm/logs/go2rtc",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "83": {
      "cmdline": "s6-log -b -- T 1 n0 s10000000 T /dev/shm/logs/frigate",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "90": {
      "cmdline": "/usr/local/go2rtc/bin/go2rtc -config=/dev/shm/go2rtc.yaml",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.7"
    },
    "101": {
      "cmdline": "bash ./run.user go2rtc-healthcheck",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "113": {
      "cmdline": "python3 -u -m frigate",
      "cpu": "1.3",
      "cpu_average": "1",
      "mem": "10.9"
    },
    "121": {
      "cmdline": "nginx: master process nginx",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.1"
    },
    "146": {
      "cmdline": "nginx: worker process",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.3"
    },
    "147": {
      "cmdline": "nginx: worker process",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.3"
    },
    "148": {
      "cmdline": "nginx: worker process",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.6"
    },
    "149": {
      "cmdline": "nginx: worker process",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.2"
    },
    "157": {
      "cmdline": "nginx: cache manager process",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    },
    "297": {
      "cmdline": "frigate.logger       ",
      "cpu": "0.1",
      "cpu_average": "0",
      "mem": "0.6"
    },
    "308": {
      "cmdline": "/usr/bin/python3 -c from multiprocessing.resource_tracker import main;main(49)",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.1"
    },
    "309": {
      "cmdline": "frigate.detector.coral",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.9"
    },
    "310": {
      "cmdline": "frigate.output       ",
      "cpu": "0.1",
      "cpu_average": "0",
      "mem": "1.6"
    },
    "314": {
      "cmdline": "ffmpeg -f rawvideo -pix_fmt yuv420p -video_size 1280x720 -i pipe: -f mpegts -s 1280x720 -codec:v mpeg1video -q 8 -bf 0 pipe:",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.2"
    },
    "315": {
      "cmdline": "ffmpeg -f rawvideo -pix_fmt yuv420p -video_size 1920x1080 -i pipe: -f mpegts -s 1280x720 -codec:v mpeg1video -q 8 -bf 0 pipe:",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.1"
    },
    "316": {
      "cmdline": "ffmpeg -f rawvideo -pix_fmt yuv420p -video_size 1280x720 -i pipe: -f mpegts -s 1280x720 -codec:v mpeg1video -q 8 -bf 0 pipe:",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.1"
    },
    "317": {
      "cmdline": "ffmpeg -f rawvideo -pix_fmt yuv420p -video_size 3840x2160 -i pipe: -f mpegts -s 3840x2160 -codec:v mpeg1video -q 10 -bf 0 pipe:",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "2.9"
    },
    "341": {
      "cmdline": "frigate.process:gate ",
      "cpu": "0.0",
      "cpu_average": "1",
      "mem": "1.6"
    },
    "342": {
      "cmdline": "frigate.process:front",
      "cpu": "0.0",
      "cpu_average": "1",
      "mem": "1.9"
    },
    "343": {
      "cmdline": "frigate.process:backyard",
      "cpu": "0.0",
      "cpu_average": "1",
      "mem": "1.6"
    },
    "346": {
      "cmdline": "frigate.capture:gate ",
      "cpu": "0.2",
      "cpu_average": "0",
      "mem": "1.2"
    },
    "348": {
      "cmdline": "frigate.capture:front",
      "cpu": "0.1",
      "cpu_average": "1",
      "mem": "1.3"
    },
    "354": {
      "cmdline": "frigate.capture:backyard",
      "cpu": "0.2",
      "cpu_average": "0",
      "mem": "1.2"
    },
    "955596": {
      "cmdline": "sleep 30s",
      "cpu": "0.0",
      "cpu_average": "0",
      "mem": "0.0"
    }
  },
  "detection_fps": 12.0,
  "detectors": {
    "coral": {
      "detection_start": 0.0,
      "inference_speed": 4.96,
      "pid": 309
    }
  },
  "gpu_usages": {
    "intel-vaapi": {
      "gpu": "0.0%",
      "mem": "-%"
    }
  },
  "processes": {
    "go2rtc": {
      "pid": 90
    },
    "logger": {
      "pid": 297
    },
    "recording": {
      "pid": 299
    }
  },
  "service": {
    "last_updated": 1706575723,
    "latest_version": "0.12.1",
    "storage": {
      "/dev/shm": {
        "free": 310.1,
        "mount_type": "tmpfs",
        "total": 322.0,
        "used": 11.9
      },
      "/media/frigate/clips": {
        "free": 1029069.5,
        "mount_type": "ext4",
        "total": 3754444.8,
        "used": 2534586.9
      },
      "/media/frigate/recordings": {
        "free": 1029069.5,
        "mount_type": "ext4",
        "total": 3754444.8,
        "used": 2534586.9
      },
      "/tmp/cache": {
        "free": 0.0,
        "mount_type": "tmpfs",
        "total": 5722.0,
        "used": 5722.0
      }
    },
    "temperatures": {
      "apex_0": 39.05
    },
    "uptime": 344053,
    "version": "0.13.0-614a36a"
  }
}

Operating system

Proxmox

Install method

Docker Compose

Coral version

PCIe

Any other information that may be helpful

No response

blakeblackshear / frigate