valentinfrlch
commented
9 hours ago LLM Vision accesses your camera feeds using home assistant's url. You can change it in Settings>System>Network. Auto should work, but in case it doesn't, try changing it to the local IP:
Bug Description
This automation was created from the Blueprint.
Trace shows consistent and repeatable error across various cameras using Anthropic (only LLM tested).
The trace looks like it's attempting to pull images from the binary sensor instead of the camera. I have rebuilt the automation multiple times, and re-setup the LLM provider as well. No luck.
1.3.1
Service Call
Logs
Logger: homeassistant.components.automation.ai_motion Source: components/automation/init.py:743 integration: Automation (documentation, issues) First occurred: November 23, 2024 at 3:36:12 PM (69 occurrences) Last logged: 10:44:08 AM
While executing automation automation.ai_motion Traceback (most recent call last): File "/usr/src/homeassistant/homeassistant/components/automation/init.py", line 743, in async_trigger return await self.action_script.async_run( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 1801, in async_run return await asyncio.shield(create_eager_task(run.async_run())) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 464, in async_run await self._async_step(log_exceptions=False) File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 528, in _async_step self._handle_exception( File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 558, in _handle_exception raise exception File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 526, in _async_step await getattr(self, handler)() File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 1049, in _async_choose_step await self._async_run_script(script) File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 1274, in _async_run_script result = await self._async_run_long_action( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 727, in _async_run_long_action return await long_task ^^^^^^^^^^^^^^^ File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 1801, in async_run return await asyncio.shield(create_eager_task(run.async_run())) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 464, in async_run await self._async_step(log_exceptions=False) File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 528, in _async_step self._handle_exception( File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 558, in _handle_exception raise exception File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 526, in _async_step await getattr(self, handler)() File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 764, in _async_call_service_step response_data = await self._async_run_long_action( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/src/homeassistant/homeassistant/helpers/script.py", line 727, in _async_run_long_action return await long_task ^^^^^^^^^^^^^^^ File "/usr/src/homeassistant/homeassistant/core.py", line 2802, in async_call response_data = await coro ^^^^^^^^^^ File "/usr/src/homeassistant/homeassistant/core.py", line 2845, in _execute_service return await target(service_call) ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/config/custom_components/llmvision/init.py", line 305, in stream_analyzer client = await processor.add_streams(image_entities=call.image_entities, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/config/custom_components/llmvision/media_handlers.py", line 428, in add_streams await self.record( File "/config/custom_components/llmvision/media_handlers.py", line 220, in record await asyncio.gather(*(record_camera(image_entity, image_entities.index(image_entity)) for image_entity in image_entities)) File "/config/custom_components/llmvision/media_handlers.py", line 162, in record_camera frame_url = base_url + \ ^^^^^^^^^^^^ TypeError: can only concatenate str (not "NoneType") to str
{ "trace": { "last_step": "action/3/choose/1/sequence/0", "run_id": "6dfc94183c5b3d68e51b6234795b4a57", "state": "stopped", "script_execution": "error", "timestamp": { "start": "2024-11-23T18:40:28.961315+00:00", "finish": "2024-11-23T18:40:28.998304+00:00" }, "domain": "automation", "item_id": "1732307860390", "error": "can only concatenate str (not \"NoneType\") to str", "trigger": "state of binary_sensor.front_door_camera_motion", "trace": { "trigger/2": [ { "path": "trigger/2", "timestamp": "2024-11-23T18:40:28.963534+00:00", "changed_variables": { "this": { "entity_id": "automation.ai_motion", "state": "on", "attributes": { "id": "1732307860390", "last_triggered": "2024-11-23T18:28:33.576168+00:00", "mode": "single", "current": 0, "friendly_name": "AI Motion" }, "last_changed": "2024-11-23T18:20:44.619956+00:00", "last_reported": "2024-11-23T18:28:33.601701+00:00", "last_updated": "2024-11-23T18:28:33.601701+00:00", "context": { "id": "01JDD4GRN53AJEX0W9JE46H84Y", "parent_id": "01JDD4GRN47A6A63NRXJSSADMJ", "user_id": null } }, "trigger": { "id": "motion_sensor_trigger", "idx": "2", "alias": null, "platform": "state", "entity_id": "binary_sensor.front_door_camera_motion", "from_state": { "entity_id": "binary_sensor.front_door_camera_motion", "state": "off", "attributes": { "friendly_name": "Front Door Camera Motion" }, "last_changed": "2024-11-23T18:34:21.771976+00:00", "last_reported": "2024-11-23T18:34:21.771976+00:00", "last_updated": "2024-11-23T18:34:21.771976+00:00", "context": { "id": "01JDD4VCPB69X9AG71NSTZTWZK", "parent_id": null, "user_id": null } }, "to_state": { "entity_id": "binary_sensor.front_door_camera_motion", "state": "on", "attributes": { "friendly_name": "Front Door Camera Motion" }, "last_changed": "2024-11-23T18:40:28.960042+00:00", "last_reported": "2024-11-23T18:40:28.960042+00:00", "last_updated": "2024-11-23T18:40:28.960042+00:00", "context": { "id": "01JDD56K90D0E18MXGGBP10QMD", "parent_id": null, "user_id": null } }, "for": null, "attribute": null, "description": "state of binary_sensor.front_door_camera_motion" }, "important": false, "cooldown": 10, "mode": "Camera", "preview_mode": "Snapshot", "notify_devices": [ "7716d85f5654913d7afc3977d00c3b7f" ], "device_name_map": [ "mobile_app_peters_iphone" ], "camera_entities_list": [ "camera.nhsky_front_door" ], "motion_sensors_list": [ "binary_sensor.front_door_camera_motion" ], "camera_entity": "binary_sensor.front_door_camera_motion", "tag": "binary_sensor.front_door_camera_motion1732387228", "group": "binary_sensor.front_door_camera_motion", "label": "Motion detected", "camera": "Binary sensor.front door camera motion", "video": "", "image": "/local/llmvision/binary_sensor.front_door_camera_motion_0.jpg", "importance_prompt": "Your job is to classify security events based on cctv footage. Your options: \"passive\" if an event seems unimportant, \"time-sensitive\" if important and \"critical\" for suspicious events. Use \"critical\" only for possible burglaries and similar events. \"time-sensitive\" could be a courier at the front door or an event of similar importance. Reply with these replies exactly.\n" } } ], "condition/0": [ { "path": "condition/0", "timestamp": "2024-11-23T18:40:28.963618+00:00", "result": { "result": true, "entities": [ "automation.ai_motion" ] } } ], "action/0": [ { "path": "action/0", "timestamp": "2024-11-23T18:40:28.964606+00:00", "changed_variables": { "context": { "id": "01JDD56K916M3FJPF4DTV0D927", "parent_id": "01JDD56K90D0E18MXGGBP10QMD", "user_id": null } } } ], "action/0/choose/0": [ { "path": "action/0/choose/0", "timestamp": "2024-11-23T18:40:28.964755+00:00", "result": { "result": false } } ], "action/0/choose/0/conditions/0": [ { "path": "action/0/choose/0/conditions/0", "timestamp": "2024-11-23T18:40:28.964827+00:00", "result": { "result": false, "entities": [] } } ], "action/1": [ { "path": "action/1", "timestamp": "2024-11-23T18:40:28.965167+00:00" } ], "action/1/choose/0": [ { "path": "action/1/choose/0", "timestamp": "2024-11-23T18:40:28.965275+00:00", "result": { "result": false } } ], "action/1/choose/0/conditions/0": [ { "path": "action/1/choose/0/conditions/0", "timestamp": "2024-11-23T18:40:28.965337+00:00", "result": { "result": false, "entities": [] } } ], "action/2": [ { "path": "action/2", "timestamp": "2024-11-23T18:40:28.965617+00:00", "result": { "choice": 0 } } ], "action/2/choose/0": [ { "path": "action/2/choose/0", "timestamp": "2024-11-23T18:40:28.965708+00:00", "result": { "result": true } } ], "action/2/choose/0/conditions/0": [ { "path": "action/2/choose/0/conditions/0", "timestamp": "2024-11-23T18:40:28.965761+00:00", "result": { "result": true, "entities": [] } } ], "action/2/choose/0/sequence/0": [ { "path": "action/2/choose/0/sequence/0", "timestamp": "2024-11-23T18:40:28.966155+00:00" } ], "action/2/choose/0/sequence/0/repeat/sequence/0": [ { "path": "action/2/choose/0/sequence/0/repeat/sequence/0", "timestamp": "2024-11-23T18:40:28.967651+00:00", "changed_variables": { "repeat": { "first": true, "index": 1, "last": true, "item": "mobile_app_peters_iphone" } }, "result": { "params": { "domain": "notify", "service": "mobile_app_peters_iphone", "service_data": { "title": "Motion detected", "message": "Binary sensor.front door camera motion", "data": { "video": null, "image": "/local/llmvision/binary_sensor.front_door_camera_motion_0.jpg", "entity_id": "", "url": "/lovelace/0", "clickAction": "/lovelace/0", "tag": "binary_sensor.front_door_camera_motion1732387228", "group": "binary_sensor.front_door_camera_motion", "interruption-level": "active" } }, "target": {} }, "running_script": false } } ], "action/3": [ { "path": "action/3", "timestamp": "2024-11-23T18:40:28.969972+00:00", "error": "can only concatenate str (not \"NoneType\") to str", "result": { "choice": 1 } } ], "action/3/choose/0": [ { "path": "action/3/choose/0", "timestamp": "2024-11-23T18:40:28.970108+00:00", "result": { "result": false } } ], "action/3/choose/0/conditions/0": [ { "path": "action/3/choose/0/conditions/0", "timestamp": "2024-11-23T18:40:28.970169+00:00", "result": { "result": false, "entities": [] } } ], "action/3/choose/1": [ { "path": "action/3/choose/1", "timestamp": "2024-11-23T18:40:28.970444+00:00", "result": { "result": true } } ], "action/3/choose/1/conditions/0": [ { "path": "action/3/choose/1/conditions/0", "timestamp": "2024-11-23T18:40:28.970504+00:00", "result": { "result": true, "entities": [] } } ], "action/3/choose/1/sequence/0": [ { "path": "action/3/choose/1/sequence/0", "timestamp": "2024-11-23T18:40:28.970923+00:00", "error": "can only concatenate str (not \"NoneType\") to str", "result": { "params": { "domain": "llmvision", "service": "stream_analyzer", "service_data": { "image_entity": [ "binary_sensor.front_door_camera_motion" ], "duration": 1, "provider": "01JAGQTC733R5QXHYFKABY6Q5K", "model": "claude-3-5-sonnet-20241022", "message": "Summarize what's happening in the camera feed (one sentence max). Don't describe the scene! If there is a person, describe what they're doing and what they look like. If they look like a courier mention that! If nothing is happening, say so.", "remember": false, "include_filename": true, "max_frames": 1, "target_width": 1024, "detail": "low", "max_tokens": 40, "temperature": 0.1, "expose_images": true }, "target": {} }, "running_script": false } } ] }, "config": { "variables": { "important": false, "cooldown": 10, "mode": "Camera", "preview_mode": "Snapshot", "notify_devices": [ "7716d85f5654913d7afc3977d00c3b7f" ], "device_name_map": "{% set ns = namespace(device_names=[]) %} {% for device_id in notify_devices %}\n {% set device_name = device_attr(device_id, \"name\") %}\n {% set sanitized_name = \"mobileapp\" + device_name | lower | regexreplace(\"[^a-z0-9 ]\", \"\") | replace(\" \", \"\") %}\n {% set ns.device_names = ns.device_names + [sanitized_name] %}\n{% endfor %} {{ ns.device_names }}\n", "camera_entities_list": [ "camera.nhsky_front_door" ], "motion_sensors_list": [ "binary_sensor.front_door_camera_motion" ], "camera_entity": "{% if mode == 'Camera' %}\n {% if motion_sensors_list and not \"camera\" in trigger.entity_id %}\n {% set index = motion_sensors_list.index(trigger.entity_id) %}\n {{ camera_entities_list[index] }}\n {% else %}\n {{ trigger.entity_id }}\n {% endif %}\n{% else %}\n {{ trigger.payload_json['after']['camera'] }}\n{% endif %}\n", "tag": "{% if mode == 'Frigate' %}\n {{ trigger.payload_json['after']['camera'] + int(as_timestamp(now()))|string }}\n{% else %}\n {{ camera_entity + int(as_timestamp(now()))|string }}\n{% endif %}\n", "group": "{% if mode == 'Frigate' %}\n {{ trigger.payload_json['after']['camera'] }}\n{% else %}\n {{ camera_entity }}\n{% endif %}\n", "label": "{% if mode == 'Frigate' %}\n {{ trigger.payload_json['after']['label']|capitalize }} seen\n{% else %}\n Motion detected\n{% endif %}\n", "camera": "{% if mode == 'Frigate' %}\n {{ trigger.payloadjson['after']['camera'].replace('', ' ')|capitalize }}\n{% else %}\n {{ cameraentity.replace(\"camera.\", \"\").replace(\"\", \" \")|capitalize }}\n{% endif %}\n", "video": "{% if mode == 'Frigate' %}\n /api/frigate/notifications/{{ trigger.payload_json['after']['id'] }}/clip.mp4\n{% else %} {% endif %}\n", "image": "{% if mode == 'Frigate' %}\n ''\n{% else %}\n {% if preview_mode == 'Live Preview' %}\n {{ '/api/camera_proxy/' + camera_entity }}\n {% else %}\n /local/llmvision/{{camera_entity.replace(\"camera.\", \"\")}}_0.jpg\n {% endif %}\n{% endif %}\n", "importance_prompt": "Your job is to classify security events based on cctv footage. Your options: \"passive\" if an event seems unimportant, \"time-sensitive\" if important and \"critical\" for suspicious events. Use \"critical\" only for possible burglaries and similar events. \"time-sensitive\" could be a courier at the front door or an event of similar importance. Reply with these replies exactly.\n" }, "triggers": [ { "platform": "mqtt", "topic": "frigate/events", "id": "frigate_trigger" }, { "platform": "state", "entity_id": [ "camera.nhsky_front_door" ], "to": "recording", "id": "camera_trigger" }, { "platform": "state", "entity_id": [ "binary_sensor.front_door_camera_motion" ], "to": "on", "id": "motion_sensor_trigger" } ], "conditions": [ { "condition": "template", "value_template": "{% if mode == 'Frigate' %}\n {{ trigger.payload_json[\"type\"] == \"end\" and (state_attr(this.entity_id, 'last_triggered') is none or (now() - state_attr(this.entity_id, 'last_triggered')).total_seconds() / 60 > cooldown) and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list }}\n{% else %}\n {{ state_attr(this.entity_id, 'last_triggered') is none or (now() - state_attr(this.entity_id, 'last_triggered')).total_seconds() / 60 > cooldown }}\n{% endif %}\n" } ], "actions": [ { "choose": [ { "conditions": [ { "condition": "template", "value_template": "{{ important }}" } ], "sequence": [ { "alias": "Decide Important", "choose": [ { "conditions": [ { "condition": "template", "value_template": "{{ mode == 'Frigate'}}" } ], "sequence": [ { "action": "llmvision.image_analyzer", "data": { "image_entity": "{{ ['camera.' + trigger.payload_json['after']['camera']|lower] }}", "provider": "01JAGQTC733R5QXHYFKABY6Q5K", "model": "claude-3-5-sonnet-20241022", "message": "{{importance_prompt}}", "include_filename": true, "target_width": 1280, "detail": "low", "max_tokens": 3, "temperature": 0.1 }, "response_variable": "importance" } ] }, { "conditions": [ { "condition": "template", "value_template": "{{ mode == 'Camera' }}" } ], "sequence": [ { "action": "llmvision.image_analyzer", "data": { "image_entity": "{{[camera_entity]}}", "provider": "01JAGQTC733R5QXHYFKABY6Q5K", "model": "claude-3-5-sonnet-20241022", "message": "{{importance_prompt}}", "include_filename": true, "target_width": 1280, "detail": "low", "max_tokens": 3, "temperature": 0.1 }, "response_variable": "importance" } ] } ] } ] } ] }, { "choose": [ { "conditions": [ { "condition": "template", "value_template": "{{ importance is defined and importance.response_text|lower == 'passive' }}" } ], "sequence": [ { "stop": "Event is not important" } ] } ] }, { "choose": [ { "conditions": [ { "condition": "template", "value_template": "{{ image != '' or video != '' }}" } ], "sequence": [ { "alias": "Send instant notification to notify devices", "repeat": { "for_each": "{{device_name_map}}", "sequence": [ { "action": "notify.{{ repeat.item }}", "data": { "title": "{{ label }}", "message": "{{camera}}", "data": { "video": "{{video if video != '' else None}}", "image": "{{image if image != '' else None}}", "entity_id": "{{camera_entity if mode=='Camera' and preview_mode=='Live Preview'}}", "url": "/lovelace/0", "clickAction": "/lovelace/0", "tag": "{{tag}}", "group": "{{group}}", "interruption-level": "{{importance.response_text|lower if importance is defined else 'active'}}" } } } ] } } ] } ] }, { "alias": "Analyze event", "choose": [ { "conditions": [ { "condition": "template", "value_template": "{{ mode == 'Frigate' }}" } ], "sequence": [ { "action": "llmvision.video_analyzer", "data": { "event_id": "{{ trigger.payload_json['after']['id'] }}", "provider": "01JAGQTC733R5QXHYFKABY6Q5K", "model": "claude-3-5-sonnet-20241022", "message": "Summarize what's happening in the camera feed (one sentence max). Don't describe the scene! If there is a person, describe what they're doing and what they look like. If they look like a courier mention that! If nothing is happening, say so.", "remember": false, "include_filename": true, "max_frames": 1, "target_width": 1024, "detail": "low", "max_tokens": 40, "temperature": 0.1 }, "response_variable": "response" } ] }, { "conditions": [ { "condition": "template", "value_template": "{{ mode == 'Camera' }}" } ], "sequence": [ { "action": "llmvision.stream_analyzer", "data": { "image_entity": "{{[camera_entity]}}", "duration": 1, "provider": "01JAGQTC733R5QXHYFKABY6Q5K", "model": "claude-3-5-sonnet-20241022", "message": "Summarize what's happening in the camera feed (one sentence max). Don't describe the scene! If there is a person, describe what they're doing and what they look like. If they look like a courier mention that! If nothing is happening, say so.", "remember": false, "include_filename": true, "max_frames": 1, "target_width": 1024, "detail": "low", "max_tokens": 40, "temperature": 0.1, "expose_images": "{{true if preview_mode == 'Snapshot'}}" }, "response_variable": "response" } ] } ] }, { "choose": [ { "conditions": [ { "condition": "template", "value_template": "{{ image != '' or video != '' }}" } ], "sequence": [ { "alias": "Send instant notification to notify devices", "repeat": { "for_each": "{{device_name_map}}", "sequence": [ { "action": "notify.{{ repeat.item }}", "data": { "title": "{{ label }}", "message": "{{response.response_text}}", "data": { "video": "{{video if video != '' else None}}", "image": "{{image if image != '' else None}}", "entity_id": "{{camera_entity if mode=='Camera' and preview_mode=='Live Preview'}}", "url": "/lovelace/0", "clickAction": "/lovelace/0", "tag": "{{tag}}", "group": "{{group}}", "interruption-level": "passive" } } } ] } } ] } ] } ], "id": "1732307860390", "alias": "AI Motion", "description": "" }, "blueprint_inputs": { "id": "1732307860390", "alias": "AI Motion", "description": "", "use_blueprint": { "path": "valentinfrlch/event_summary.yaml", "input": { "mode": "Camera", "notify_device": [ "7716d85f5654913d7afc3977d00c3b7f" ], "camera_entities": [ "camera.nhsky_front_door" ], "motion_sensors": [ "binary_sensor.front_door_camera_motion" ], "preview_mode": "Snapshot", "duration": 1, "max_frames": 1, "provider": "01JAGQTC733R5QXHYFKABY6Q5K", "target_width": 1024, "max_tokens": 40, "model": "claude-3-5-sonnet-20241022" } } }, "context": { "id": "01JDD56K916M3FJPF4DTV0D927", "parent_id": "01JDD56K90D0E18MXGGBP10QMD", "user_id": null } }, "logbookEntries": [ { "name": "AI Motion", "message": "triggered by state of binary_sensor.front_door_camera_motion", "source": "state of binary_sensor.front_door_camera_motion", "entity_id": "automation.ai_motion", "context_id": "01JDD56K916M3FJPF4DTV0D927", "domain": "automation", "when": 1732387228.9642916 } ] }
alias: Analyze event choose:
Additional context