search

blakeblackshear / frigate

NVR with realtime local object detection for IP cameras
https://frigate.video
MIT License
18.41k stars 1.68k forks source link

[EdgeTPU Support]: Adding Coral TPU (USB) to frigate chart APP (TrueNAS) #5542

Closed SchoGiGIT closed 1 year ago

SchoGiGIT commented 1 year ago

Describe the problem you are having

system:

I need a little bit of help adding my coral TPU (USB version) to the frigate chart app.

When plugging in the TPU it shows up as the following in dmesg: image

In the frigate app config i need to add a host path for the coral usb: image

But i do not seem to be able to figure out what i should put here. I know it should be something under /dev/... but all guides indicate dmesg should show something like [sdb], [tty]... under dmesg for the device.

image

Version

0.11.1-2EADA21

Frigate config file

{
  "birdseye": {
    "enabled": true,
    "height": 720,
    "mode": "objects",
    "quality": 8,
    "width": 1280
  },
  "cameras": {
    "tuin": {
      "best_image_timeout": 60,
      "birdseye": {
        "enabled": true,
        "mode": "objects"
      },
      "detect": {
        "enabled": true,
        "fps": 5,
        "height": 1080,
        "max_disappeared": 25,
        "stationary": {
          "interval": 0,
          "max_frames": {
            "default": null,
            "objects": {}
          },
          "threshold": 50
        },
        "width": 1920
      },
      "ffmpeg": {
        "global_args": [
          "-hide_banner",
          "-loglevel",
          "warning"
        ],
        "hwaccel_args": [],
        "input_args": [
          "-avoid_negative_ts",
          "make_zero",
          "-fflags",
          "+genpts+discardcorrupt",
          "-rtsp_transport",
          "tcp",
          "-timeout",
          "5000000",
          "-use_wallclock_as_timestamps",
          "1"
        ],
        "inputs": [
          {
            "global_args": [],
            "hwaccel_args": [],
            "input_args": [],
            "path": "rtsp://192.168.40.56:554/user=admin&password=so4uSW5Jv4QTb2&channel=1&stream=0.sdp?real_stream",
            "roles": [
              "record",
              "rtmp",
              "detect"
            ]
          }
        ],
        "output_args": {
          "detect": [
            "-f",
            "rawvideo",
            "-pix_fmt",
            "yuv420p"
          ],
          "record": [
            "-f",
            "segment",
            "-segment_time",
            "10",
            "-segment_format",
            "mp4",
            "-reset_timestamps",
            "1",
            "-strftime",
            "1",
            "-c",
            "copy",
            "-an"
          ],
          "rtmp": [
            "-c",
            "copy",
            "-f",
            "flv"
          ]
        }
      },
      "ffmpeg_cmds": [
        {
          "cmd": "ffmpeg -hide_banner -loglevel warning -avoid_negative_ts make_zero -fflags +genpts+discardcorrupt -rtsp_transport tcp -timeout 5000000 -use_wallclock_as_timestamps 1 -i rtsp://192.168.40.56:554/user=admin&password=so4uSW5Jv4QTb2&channel=1&stream=0.sdp?real_stream -f segment -segment_time 10 -segment_format mp4 -reset_timestamps 1 -strftime 1 -c copy -an /tmp/cache/tuin-%Y%m%d%H%M%S.mp4 -c copy -f flv rtmp://127.0.0.1/live/tuin -r 5 -s 1920x1080 -f rawvideo -pix_fmt yuv420p pipe:",
          "roles": [
            "record",
            "rtmp",
            "detect"
          ]
        }
      ],
      "live": {
        "height": 1080,
        "quality": 10
      },
      "motion": {
        "contour_area": 30,
        "delta_alpha": 0.2,
        "frame_alpha": 0.2,
        "frame_height": 50,
        "improve_contrast": false,
        "mask": "",
        "mqtt_off_delay": 30,
        "threshold": 25
      },
      "mqtt": {
        "bounding_box": true,
        "crop": true,
        "enabled": true,
        "height": 270,
        "quality": 70,
        "required_zones": [],
        "timestamp": true
      },
      "name": "tuin",
      "objects": {
        "filters": {
          "car": {
            "mask": null,
            "max_area": 24000000,
            "max_ratio": 24000000,
            "min_area": 0,
            "min_ratio": 0,
            "min_score": 0.5,
            "threshold": 0.7
          },
          "cat": {
            "mask": null,
            "max_area": 24000000,
            "max_ratio": 24000000,
            "min_area": 0,
            "min_ratio": 0,
            "min_score": 0.5,
            "threshold": 0.7
          },
          "dog": {
            "mask": null,
            "max_area": 24000000,
            "max_ratio": 24000000,
            "min_area": 0,
            "min_ratio": 0,
            "min_score": 0.5,
            "threshold": 0.7
          },
          "person": {
            "mask": null,
            "max_area": 24000000,
            "max_ratio": 24000000,
            "min_area": 0,
            "min_ratio": 0,
            "min_score": 0.5,
            "threshold": 0.7
          }
        },
        "mask": "",
        "track": [
          "cat",
          "dog",
          "car",
          "person"
        ]
      },
      "record": {
        "enabled": false,
        "events": {
          "objects": null,
          "post_capture": 5,
          "pre_capture": 5,
          "required_zones": [],
          "retain": {
            "default": 2,
            "mode": "motion",
            "objects": {}
          }
        },
        "expire_interval": 60,
        "retain": {
          "days": 1,
          "mode": "all"
        },
        "retain_days": null
      },
      "rtmp": {
        "enabled": true
      },
      "snapshots": {
        "bounding_box": true,
        "clean_copy": true,
        "crop": false,
        "enabled": true,
        "height": null,
        "quality": 70,
        "required_zones": [],
        "retain": {
          "default": 10,
          "mode": "motion",
          "objects": {}
        },
        "timestamp": false
      },
      "timestamp_style": {
        "color": {
          "blue": 255,
          "green": 255,
          "red": 255
        },
        "effect": null,
        "format": "%m/%d/%Y %H:%M:%S",
        "position": "tl",
        "thickness": 2
      },
      "ui": {
        "dashboard": true,
        "order": 0
      },
      "zones": {}
    },
    "woonk": {
      "best_image_timeout": 60,
      "birdseye": {
        "enabled": true,
        "mode": "objects"
      },
      "detect": {
        "enabled": true,
        "fps": 5,
        "height": 1080,
        "max_disappeared": 25,
        "stationary": {
          "interval": 0,
          "max_frames": {
            "default": null,
            "objects": {}
          },
          "threshold": 50
        },
        "width": 1920
      },
      "ffmpeg": {
        "global_args": [
          "-hide_banner",
          "-loglevel",
          "warning"
        ],
        "hwaccel_args": [],
        "input_args": [
          "-avoid_negative_ts",
          "make_zero",
          "-fflags",
          "+genpts+discardcorrupt",
          "-rtsp_transport",
          "tcp",
          "-timeout",
          "5000000",
          "-use_wallclock_as_timestamps",
          "1"
        ],
        "inputs": [
          {
            "global_args": [],
            "hwaccel_args": [],
            "input_args": [],
            "path": "rtsp://192.168.40.43:554/user=admin&password=FPZFCRDs&channel=1&stream=0.sdp?real_stream",
            "roles": [
              "record",
              "rtmp",
              "detect"
            ]
          }
        ],
        "output_args": {
          "detect": [
            "-f",
            "rawvideo",
            "-pix_fmt",
            "yuv420p"
          ],
          "record": [
            "-f",
            "segment",
            "-segment_time",
            "10",
            "-segment_format",
            "mp4",
            "-reset_timestamps",
            "1",
            "-strftime",
            "1",
            "-c",
            "copy",
            "-an"
          ],
          "rtmp": [
            "-c",
            "copy",
            "-f",
            "flv"
          ]
        }
      },
      "ffmpeg_cmds": [
        {
          "cmd": "ffmpeg -hide_banner -loglevel warning -avoid_negative_ts make_zero -fflags +genpts+discardcorrupt -rtsp_transport tcp -timeout 5000000 -use_wallclock_as_timestamps 1 -i rtsp://192.168.40.43:554/user=admin&password=FPZFCRDs&channel=1&stream=0.sdp?real_stream -f segment -segment_time 10 -segment_format mp4 -reset_timestamps 1 -strftime 1 -c copy -an /tmp/cache/woonk-%Y%m%d%H%M%S.mp4 -c copy -f flv rtmp://127.0.0.1/live/woonk -r 5 -s 1920x1080 -f rawvideo -pix_fmt yuv420p pipe:",
          "roles": [
            "record",
            "rtmp",
            "detect"
          ]
        }
      ],
      "live": {
        "height": 1080,
        "quality": 10
      },
      "motion": {
        "contour_area": 30,
        "delta_alpha": 0.2,
        "frame_alpha": 0.2,
        "frame_height": 50,
        "improve_contrast": false,
        "mask": "",
        "mqtt_off_delay": 30,
        "threshold": 25
      },
      "mqtt": {
        "bounding_box": true,
        "crop": true,
        "enabled": true,
        "height": 270,
        "quality": 70,
        "required_zones": [],
        "timestamp": true
      },
      "name": "woonk",
      "objects": {
        "filters": {
          "car": {
            "mask": null,
            "max_area": 24000000,
            "max_ratio": 24000000,
            "min_area": 0,
            "min_ratio": 0,
            "min_score": 0.5,
            "threshold": 0.7
          },
          "cat": {
            "mask": null,
            "max_area": 24000000,
            "max_ratio": 24000000,
            "min_area": 0,
            "min_ratio": 0,
            "min_score": 0.5,
            "threshold": 0.7
          },
          "dog": {
            "mask": null,
            "max_area": 24000000,
            "max_ratio": 24000000,
            "min_area": 0,
            "min_ratio": 0,
            "min_score": 0.5,
            "threshold": 0.7
          },
          "person": {
            "mask": null,
            "max_area": 24000000,
            "max_ratio": 24000000,
            "min_area": 0,
            "min_ratio": 0,
            "min_score": 0.5,
            "threshold": 0.7
          }
        },
        "mask": "",
        "track": [
          "cat",
          "dog",
          "car",
          "person"
        ]
      },
      "record": {
        "enabled": false,
        "events": {
          "objects": null,
          "post_capture": 5,
          "pre_capture": 5,
          "required_zones": [],
          "retain": {
            "default": 2,
            "mode": "motion",
            "objects": {}
          }
        },
        "expire_interval": 60,
        "retain": {
          "days": 1,
          "mode": "all"
        },
        "retain_days": null
      },
      "rtmp": {
        "enabled": true
      },
      "snapshots": {
        "bounding_box": true,
        "clean_copy": true,
        "crop": false,
        "enabled": true,
        "height": null,
        "quality": 70,
        "required_zones": [],
        "retain": {
          "default": 10,
          "mode": "motion",
          "objects": {}
        },
        "timestamp": false
      },
      "timestamp_style": {
        "color": {
          "blue": 255,
          "green": 255,
          "red": 255
        },
        "effect": null,
        "format": "%m/%d/%Y %H:%M:%S",
        "position": "tl",
        "thickness": 2
      },
      "ui": {
        "dashboard": true,
        "order": 0
      },
      "zones": {}
    }
  },
  "database": {
    "path": "/db/frigate.db"
  },
  "detect": {
    "enabled": true,
    "fps": 5,
    "height": 1080,
    "max_disappeared": null,
    "stationary": {
      "interval": 0,
      "max_frames": {
        "default": null,
        "objects": {}
      },
      "threshold": null
    },
    "width": 1920
  },
  "detectors": {
    "cpu": {
      "device": "usb",
      "num_threads": 3,
      "type": "cpu"
    }
  },
  "environment_vars": {},
  "ffmpeg": {
    "global_args": [
      "-hide_banner",
      "-loglevel",
      "warning"
    ],
    "hwaccel_args": [],
    "input_args": [
      "-avoid_negative_ts",
      "make_zero",
      "-fflags",
      "+genpts+discardcorrupt",
      "-rtsp_transport",
      "tcp",
      "-timeout",
      "5000000",
      "-use_wallclock_as_timestamps",
      "1"
    ],
    "output_args": {
      "detect": [
        "-f",
        "rawvideo",
        "-pix_fmt",
        "yuv420p"
      ],
      "record": [
        "-f",
        "segment",
        "-segment_time",
        "10",
        "-segment_format",
        "mp4",
        "-reset_timestamps",
        "1",
        "-strftime",
        "1",
        "-c",
        "copy",
        "-an"
      ],
      "rtmp": [
        "-c",
        "copy",
        "-f",
        "flv"
      ]
    }
  },
  "live": {
    "height": 1080,
    "quality": 10
  },
  "logger": {
    "default": "info",
    "logs": {}
  },
  "model": {
    "height": 320,
    "labelmap": {},
    "labelmap_path": null,
    "path": null,
    "width": 320
  },
  "motion": null,
  "mqtt": {
    "client_id": "frigate",
    "host": "192.168.69.10",
    "password": null,
    "port": 1883,
    "stats_interval": 60,
    "tls_ca_certs": null,
    "tls_client_cert": null,
    "tls_client_key": null,
    "tls_insecure": null,
    "topic_prefix": "frigate",
    "user": null
  },
  "objects": {
    "filters": null,
    "mask": "",
    "track": [
      "cat",
      "dog",
      "car",
      "person"
    ]
  },
  "plus": {
    "enabled": false
  },
  "record": {
    "enabled": true,
    "events": {
      "objects": null,
      "post_capture": 5,
      "pre_capture": 5,
      "required_zones": [],
      "retain": {
        "default": 2,
        "mode": "motion",
        "objects": {}
      }
    },
    "expire_interval": 60,
    "retain": {
      "days": 1,
      "mode": "all"
    },
    "retain_days": null
  },
  "rtmp": {
    "enabled": true
  },
  "snapshots": {
    "bounding_box": true,
    "clean_copy": true,
    "crop": false,
    "enabled": false,
    "height": null,
    "quality": 70,
    "required_zones": [],
    "retain": {
      "default": 10,
      "mode": "motion",
      "objects": {}
    },
    "timestamp": false
  },
  "timestamp_style": {
    "color": {
      "blue": 255,
      "green": 255,
      "red": 255
    },
    "effect": null,
    "format": "%m/%d/%Y %H:%M:%S",
    "position": "tl",
    "thickness": 2
  },
  "ui": {
    "use_experimental": false
  }
}

docker-compose file or Docker CLI command

chart app, not available

Relevant log output

not relevant

Operating system

Other

Install method

Docker Compose

Coral version

USB

Any other information that may be helpful

No response

NickM-27 commented 1 year ago

I wouldn't pass it that way. I'd also suggest just passing the entire /dev/bus/usb to the container. You're going to run into problems passing a single device because the usb coral changes it's id and name after it initializes

SchoGiGIT commented 1 year ago

Wouldnt that cause problems for my mouse and keyboard connected to the host for emergencies?

NickM-27 commented 1 year ago

Why would it cause problems for the mouse and keyboard?

SchoGiGIT commented 1 year ago

I thought when i passed through the bus the host or other vms couldnt acces the usb devices anymore?

When i pass it through using this path it starts but crashes after two minutes. (does not matter if i dfine both the edgtpu or also a cpu detector) [2023-02-19 16:04:21] frigate.edgetpu ERROR : No EdgeTPU was detected. If you do not have a Coral device yet, you must configure CPU detectors.

the weirdest part is the way the chart app defines threads for a tpu? i cannot influence this... "detectors": { "cpu": { "device": "usb", "num_threads": 2, "type": "cpu" }, "edgetpu1": { "device": "usb", "num_threads": 3, "type": "edgetpu" }

NickM-27 commented 1 year ago

I thought when i passed through the bus the host or other vms couldnt acces the usb devices anymore?

That's not the case for usb bus, the frigate container won't be accessing those devices so it won't affect other clients trying to access them.

When i pass it through using this path it starts but crashes after two minutes. (does not matter if i dfine both the edgtpu or also a cpu detector)

[2023-02-19 16:04:21] frigate.edgetpu ERROR : No EdgeTPU was detected. If you do not have a Coral device yet, you must configure CPU detectors.

Right, it's not seeing the correct device. You can run lsusb in the container to see what the container sees.

the weirdest part is the way the chart app defines threads for a tpu? i cannot influence this...

"detectors": {

"cpu": {

  "device": "usb",

  "num_threads": 2,

  "type": "cpu"

},

"edgetpu1": {

  "device": "usb",

  "num_threads": 3,

  "type": "edgetpu"

}

You're just looking at the debug config. It will always have a threads value because that's apart of the config in general, but it is not used for anything.

SchoGiGIT commented 1 year ago

image it sees the mouse and keyboard at least, the nameless one is the tpu i think then?

NickM-27 commented 1 year ago

You're probably going to want to run the udev rules as described in the coral docs to make sure the name, id match

SchoGiGIT commented 1 year ago

So i followed this guide and everything seems to install well. https://coral.ai/docs/accelerator/get-started/#requirements I installed this using the shell in the docker container itself. first problem i noticed: if i restart the container all the packages are gone again (even usbutils, curl...) i dont know if this is normal? I installed them again and replugged the TPU as stated in the guide. The TPU still does not show up in the container but i get these errors on the truenas host cap1

Looking at other fora makes me think they need to be installed on the host. But similar fora also state that this should never be done on an "appliance" like truenas. what to do with this?

NickM-27 commented 1 year ago

So i followed this guide and everything seems to install well. https://coral.ai/docs/accelerator/get-started/#requirements I installed this using the shell in the docker container itself. first problem i noticed: if i restart the container all the packages are gone again (even usbutils, curl...) i dont know if this is normal?

yes it is normal, you can not edit a container unless you do bind mounts. It also doesn't make sense though because as long as the device is correct on the host it will be passed correctly to the container.

I installed them again and replugged the TPU as stated in the guide. The TPU still does not show up in the container but i get these errors on the truenas host cap1

Looking at other fora makes me think they need to be installed on the host. But similar fora also state that this should never be done on an "appliance" like truenas. what to do with this?

You may want to see what other users in the issues here have been doing with truenas, but it definitely needs to be done on the host as far as I am concerned, that's how it has worked for all the OSs I have run a usb coral on (UnRaid, Ubuntu, Debian)

SchoGiGIT commented 1 year ago

Thanks a lot for your support, i am crosposting this to the truenas fora. If a solution is reached there i will post it here for anyone else comming acros this and close the issue . Judging by the interest the reddit post got a lot of people will be following my footsteps. Still thank a lot for the awsome piece of software you guys are making, i will be making a donation later! The only to be improved thing i see is the, for an average person/hobbyist not a software engineer, steep learning curve to set it up.

SchoGiGIT commented 1 year ago

So this thing got even less understandable and i am going mad over it. I installed the drivers on the host and the tpu is detected OK lsusb host

In the frigate container i can no matter what make it being detected correctly. lsusb frigate

Even weirder, a reddit user also using the chart frigate app states he did not need to install driver. They were included in the docker container. https://www.reddit.com/r/truenas/comments/1168n6e/comment/j9ks5y2/?context=3

I am not able to make anything of this. Also i do not seem to be able to find a channel to ask specific question/find documentation for the frigate chart app. Should these be asked here, do you guys also handle the development of these apps?

NickM-27 commented 1 year ago

The chart app is not maintained by us and is entirely separate (as far as I know). Either way it is not this repo for sure so the issue would need to be placed elsewhere.

The runtime is included in the docker container, are you still getting a crash?

What happens if you do

detectors:
  coral:
    type: edgetpu
    device: usb:0

as your detectors config? what are the logs?

SchoGiGIT commented 1 year ago

Well i tried a lot of things, reinstalling the truenas app, reinstqlled the tpu drivers, tried different configs.... and nothing seemed to work. I left it how it was for a couple of days and randomly when i tried again today it seemed to work with the tpu. I started to look into what happend and found out the following:

When you install the TPU driver on the Truenas host according to the following link: https://coral.ai/docs/accelerator/get-started/#requirements

Plug in the TPU afterwards, and it will show up as "global unichip corp" in lsusb on the host. When you then try and start frigate with the TPU in the config it will crash with the following log: frigate crash.log (it seems like something went wrong with saving this log, it is the wrong one, i will updqte it with the correct one)

When you just restart frigate without doing anything else it will start fine with the folliwing log: frigate pass.log

When you no look at lsusb on the host the TPU will show up as "google inc." In the container it will at all times show up nameless..

So it work but something seems weird here.

NickM-27 commented 1 year ago

This is the expected behavior for the USB coral. The first time you try to access it it will load the internal driver (the driver is kept inside the USB coral, there is no host driver for the USB coral). After the first access and the drier is loaded, it will work as expected

github-actions[bot] commented 1 year ago

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.