Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/valentinfrlch/ha-llmvision
Browse files Browse the repository at this point in the history
  • Loading branch information
@valentinfrlch
valentinfrlch committed Jan 18, 2025
2 parents c39af35 + 7f6c617 commit dc9bd34
Show file tree
Hide file tree
Showing 5 changed files with 231 additions and 96 deletions.
11 changes: 8 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
<a href="#support">☕ Support</a>
</p>
<p align="center">
<a href="https://valentinfrlch.github.io/llmvision/"> Visit Website →</a>
<a href="https://llmvision.org"> Visit Website →</a>
</p>
<br>
<br>
Expand All @@ -40,11 +40,16 @@
</p>

## Features
- Compatible with OpenAI, Anthropic Claude, Google Gemini, AWS Bedrock (Nova & Anthropic Claude), Groq, [LocalAI](https://github.com/mudler/LocalAI), [Ollama](https://ollama.com/) and custom OpenAI compatible APIs
- Compatible with OpenAI, Anthropic Claude, Google Gemini, AWS Bedrock, Groq, [LocalAI](https://github.com/mudler/LocalAI), [Ollama](https://ollama.com/) and custom OpenAI compatible APIs
- Analyzes images and video files, live camera feeds and Frigate events
- Remembers Frigate events and camera motion events so you can ask about them later
- Seamlessly updates sensors based on image input

![features](https://github.com/user-attachments/assets/5edd11d6-79b9-4736-9387-8d22405c53b8)
See [website](https://llmvision.org) for more details.

<br>

## Blueprint
With the easy to use blueprint, you'll get important notifications intelligently summarized by AI from either Frigate or cameras in Home Assistant. LLM Vision can also remember events, so you can ask about them later. LLM Vision needs to be installed to use the blueprint.
<br>
Expand All @@ -58,7 +63,7 @@ With the easy to use blueprint, you'll get important notifications intelligently
## Resources
Check the docs for detailed instructions on how to set up LLM Vision and each of the supported providers, get inspiration from examples or join the discussion on the Home Assistant Community.

<a href="https://valentinfrlch.github.io/llmvision/"><img alt="Static Badge" src="https://img.shields.io/badge/website-teal?style=for-the-badge&&logoColor=white&link=https%3A%2F%2Fvalentinfrlch.github.io%2Fllmvision%2F"></a>
<a href="https://llmvision.org"><img alt="Static Badge" src="https://img.shields.io/badge/website-teal?style=for-the-badge&&logoColor=white&link=https%3A%2F%2Fvalentinfrlch.github.io%2Fllmvision%2F"></a>
<a href="https://llm-vision.gitbook.io/getting-started"><img src="https://img.shields.io/badge/Documentation-blue?style=for-the-badge&logo=gitbook&logoColor=white&color=18bcf2"/> </a><a href="https://llm-vision.gitbook.io/examples/"><img src="https://img.shields.io/badge/Examples-blue?style=for-the-badge&logo=gitbook&logoColor=black&color=39ffc2"/></a> </a><a href="https://community.home-assistant.io/t/llm-vision-let-home-assistant-see/729241"><img src="https://img.shields.io/badge/Community-blue?style=for-the-badge&logo=homeassistant&logoColor=white&color=03a9f4"/></a>


Expand Down
10 changes: 9 additions & 1 deletion custom_components/llmvision/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,15 @@ async def async_remove_entry(hass, entry):


async def async_unload_entry(hass, entry) -> bool:
unload_ok = await hass.config_entries.async_unload_platforms(entry, "calendar")
_LOGGER.debug(f"Unloading {entry.title} from hass.data")

# check if the entry is the calendar entry (has entry rentention_time)
if entry.data.get(CONF_RETENTION_TIME) is not None:
# unload the calendar
unload_ok = await hass.config_entries.async_unload_platforms(entry, ["calendar"])
else:
unload_ok = True

return unload_ok


Expand Down
6 changes: 0 additions & 6 deletions custom_components/llmvision/calendar.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,9 +195,3 @@ async def async_setup_entry(

calendar_entity = SemanticIndex(hass, config_entry)
async_add_entities([calendar_entity])


async def async_remove(self):
"""Handle removal of the entity."""
# _LOGGER.info(f"Removing calendar entity: {self._attr_name}")
await super().async_remove()
176 changes: 166 additions & 10 deletions custom_components/llmvision/config_flow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,13 @@ async def async_step_localai(self, user_input=None):
vol.Required(CONF_LOCALAI_HTTPS, default=False): bool,
})

if self.source == config_entries.SOURCE_RECONFIGURE:
# load existing configuration and add it to the dialog
self.init_info = self._get_reconfigure_entry().data
data_schema=self.add_suggested_values_to_schema(
data_schema, self.init_info
)

if user_input is not None:
# save provider to user_input
user_input["provider"] = self.init_info["provider"]
Expand All @@ -109,7 +116,15 @@ async def async_step_localai(self, user_input=None):
})
await localai.validate()
# add the mode to user_input
return self.async_create_entry(title=f"LocalAI ({user_input[CONF_LOCALAI_IP_ADDRESS]})", data=user_input)
if self.source == config_entries.SOURCE_RECONFIGURE:
# we're reconfiguring an existing config
return self.async_update_reload_and_abort(
self._get_reconfigure_entry(),
data_updates=user_input,
)
else:
# New config entry
return self.async_create_entry(title=f"LocalAI ({user_input[CONF_LOCALAI_IP_ADDRESS]})", data=user_input)
except ServiceValidationError as e:
_LOGGER.error(f"Validation failed: {e}")
return self.async_show_form(
Expand All @@ -130,6 +145,13 @@ async def async_step_ollama(self, user_input=None):
vol.Required(CONF_OLLAMA_HTTPS, default=False): bool,
})

if self.source == config_entries.SOURCE_RECONFIGURE:
# load existing configuration and add it to the dialog
self.init_info = self._get_reconfigure_entry().data
data_schema=self.add_suggested_values_to_schema(
data_schema, self.init_info
)

if user_input is not None:
# save provider to user_input
user_input["provider"] = self.init_info["provider"]
Expand All @@ -141,7 +163,15 @@ async def async_step_ollama(self, user_input=None):
})
await ollama.validate()
# add the mode to user_input
return self.async_create_entry(title=f"Ollama ({user_input[CONF_OLLAMA_IP_ADDRESS]})", data=user_input)
if self.source == config_entries.SOURCE_RECONFIGURE:
# we're reconfiguring an existing config
return self.async_update_reload_and_abort(
self._get_reconfigure_entry(),
data_updates=user_input,
)
else:
# New config entry
return self.async_create_entry(title=f"Ollama ({user_input[CONF_OLLAMA_IP_ADDRESS]})", data=user_input)
except ServiceValidationError as e:
_LOGGER.error(f"Validation failed: {e}")
return self.async_show_form(
Expand All @@ -160,6 +190,13 @@ async def async_step_openai(self, user_input=None):
vol.Required(CONF_OPENAI_API_KEY): str,
})

if self.source == config_entries.SOURCE_RECONFIGURE:
# load existing configuration and add it to the dialog
self.init_info = self._get_reconfigure_entry().data
data_schema=self.add_suggested_values_to_schema(
data_schema, self.init_info
)

if user_input is not None:
# save provider to user_input
user_input["provider"] = self.init_info["provider"]
Expand All @@ -169,7 +206,15 @@ async def async_step_openai(self, user_input=None):
await openai.validate()
# add the mode to user_input
user_input["provider"] = self.init_info["provider"]
return self.async_create_entry(title="OpenAI", data=user_input)
if self.source == config_entries.SOURCE_RECONFIGURE:
# we're reconfiguring an existing config
return self.async_update_reload_and_abort(
self._get_reconfigure_entry(),
data_updates=user_input,
)
else:
# New config entry
return self.async_create_entry(title="OpenAI", data=user_input)
except ServiceValidationError as e:
_LOGGER.error(f"Validation failed: {e}")
return self.async_show_form(
Expand All @@ -191,6 +236,13 @@ async def async_step_azure(self, user_input=None):
vol.Required(CONF_AZURE_VERSION, default="2024-10-01-preview"): str,
})

if self.source == config_entries.SOURCE_RECONFIGURE:
# load existing configuration and add it to the dialog
self.init_info = self._get_reconfigure_entry().data
data_schema=self.add_suggested_values_to_schema(
data_schema, self.init_info
)

if user_input is not None:
# save provider to user_input
user_input["provider"] = self.init_info["provider"]
Expand All @@ -204,7 +256,15 @@ async def async_step_azure(self, user_input=None):
await azure.validate()
# add the mode to user_input
user_input["provider"] = self.init_info["provider"]
return self.async_create_entry(title="Azure", data=user_input)
if self.source == config_entries.SOURCE_RECONFIGURE:
# we're reconfiguring an existing config
return self.async_update_reload_and_abort(
self._get_reconfigure_entry(),
data_updates=user_input,
)
else:
# New config entry
return self.async_create_entry(title="Azure", data=user_input)
except ServiceValidationError as e:
_LOGGER.error(f"Validation failed: {e}")
return self.async_show_form(
Expand All @@ -223,6 +283,13 @@ async def async_step_anthropic(self, user_input=None):
vol.Required(CONF_ANTHROPIC_API_KEY): str,
})

if self.source == config_entries.SOURCE_RECONFIGURE:
# load existing configuration and add it to the dialog
self.init_info = self._get_reconfigure_entry().data
data_schema=self.add_suggested_values_to_schema(
data_schema, self.init_info
)

if user_input is not None:
# save provider to user_input
user_input["provider"] = self.init_info["provider"]
Expand All @@ -232,7 +299,15 @@ async def async_step_anthropic(self, user_input=None):
await anthropic.validate()
# add the mode to user_input
user_input["provider"] = self.init_info["provider"]
return self.async_create_entry(title="Anthropic Claude", data=user_input)
if self.source == config_entries.SOURCE_RECONFIGURE:
# we're reconfiguring an existing config
return self.async_update_reload_and_abort(
self._get_reconfigure_entry(),
data_updates=user_input,
)
else:
# New config entry
return self.async_create_entry(title="Anthropic Claude", data=user_input)
except ServiceValidationError as e:
_LOGGER.error(f"Validation failed: {e}")
return self.async_show_form(
Expand All @@ -251,6 +326,13 @@ async def async_step_google(self, user_input=None):
vol.Required(CONF_GOOGLE_API_KEY): str,
})

if self.source == config_entries.SOURCE_RECONFIGURE:
# load existing configuration and add it to the dialog
self.init_info = self._get_reconfigure_entry().data
data_schema=self.add_suggested_values_to_schema(
data_schema, self.init_info
)

if user_input is not None:
# save provider to user_input
user_input["provider"] = self.init_info["provider"]
Expand All @@ -260,7 +342,15 @@ async def async_step_google(self, user_input=None):
await google.validate()
# add the mode to user_input
user_input["provider"] = self.init_info["provider"]
return self.async_create_entry(title="Google Gemini", data=user_input)
if self.source == config_entries.SOURCE_RECONFIGURE:
# we're reconfiguring an existing config
return self.async_update_reload_and_abort(
self._get_reconfigure_entry(),
data_updates=user_input,
)
else:
# New config entry
return self.async_create_entry(title="Google Gemini", data=user_input)
except ServiceValidationError as e:
_LOGGER.error(f"Validation failed: {e}")
return self.async_show_form(
Expand All @@ -279,6 +369,13 @@ async def async_step_groq(self, user_input=None):
vol.Required(CONF_GROQ_API_KEY): str,
})

if self.source == config_entries.SOURCE_RECONFIGURE:
# load existing configuration and add it to the dialog
self.init_info = self._get_reconfigure_entry().data
data_schema=self.add_suggested_values_to_schema(
data_schema, self.init_info
)

if user_input is not None:
# save provider to user_input
user_input["provider"] = self.init_info["provider"]
Expand All @@ -287,7 +384,15 @@ async def async_step_groq(self, user_input=None):
await groq.validate()
# add the mode to user_input
user_input["provider"] = self.init_info["provider"]
return self.async_create_entry(title="Groq", data=user_input)
if self.source == config_entries.SOURCE_RECONFIGURE:
# we're reconfiguring an existing config
return self.async_update_reload_and_abort(
self._get_reconfigure_entry(),
data_updates=user_input,
)
else:
# New config entry
return self.async_create_entry(title="Groq", data=user_input)
except ServiceValidationError as e:
_LOGGER.error(f"Validation failed: {e}")
return self.async_show_form(
Expand All @@ -308,6 +413,13 @@ async def async_step_custom_openai(self, user_input=None):
vol.Required(CONF_CUSTOM_OPENAI_API_KEY): str,
})

if self.source == config_entries.SOURCE_RECONFIGURE:
# load existing configuration and add it to the dialog
self.init_info = self._get_reconfigure_entry().data
data_schema=self.add_suggested_values_to_schema(
data_schema, self.init_info
)

if user_input is not None:
# save provider to user_input
user_input["provider"] = self.init_info["provider"]
Expand All @@ -320,7 +432,15 @@ async def async_step_custom_openai(self, user_input=None):
await custom_openai.validate()
# add the mode to user_input
user_input["provider"] = self.init_info["provider"]
return self.async_create_entry(title="Custom OpenAI compatible Provider", data=user_input)
if self.source == config_entries.SOURCE_RECONFIGURE:
# we're reconfiguring an existing config
return self.async_update_reload_and_abort(
self._get_reconfigure_entry(),
data_updates=user_input,
)
else:
# New config entry
return self.async_create_entry(title="Custom OpenAI compatible Provider", data=user_input)
except ServiceValidationError as e:
_LOGGER.error(f"Validation failed: {e}")
return self.async_show_form(
Expand All @@ -338,14 +458,30 @@ async def async_step_semantic_index(self, user_input=None):
data_schema = vol.Schema({
vol.Required(CONF_RETENTION_TIME, default=7): int,
})

if self.source == config_entries.SOURCE_RECONFIGURE:
# load existing configuration and add it to the dialog
self.init_info = self._get_reconfigure_entry().data
data_schema=self.add_suggested_values_to_schema(
data_schema, self.init_info
)

if user_input is not None:
user_input["provider"] = self.init_info["provider"]

for uid in self.hass.data[DOMAIN]:
if 'retention_time' in self.hass.data[DOMAIN][uid]:
self.async_abort(reason="already_configured")
# add the mode to user_input
return self.async_create_entry(title="LLM Vision Events", data=user_input)
if self.source == config_entries.SOURCE_RECONFIGURE:
# we're reconfiguring an existing config
return self.async_update_reload_and_abort(
self._get_reconfigure_entry(),
data_updates=user_input,
)
else:
# New config entry
return self.async_create_entry(title="LLM Vision Events", data=user_input)

return self.async_show_form(
step_id="semantic_index",
Expand All @@ -360,6 +496,13 @@ async def async_step_aws_bedrock(self, user_input=None):
vol.Required(CONF_AWS_SECRET_ACCESS_KEY): str,
})

if self.source == config_entries.SOURCE_RECONFIGURE:
# load existing configuration and add it to the dialog
self.init_info = self._get_reconfigure_entry().data
data_schema=self.add_suggested_values_to_schema(
data_schema, self.init_info
)

if user_input is not None:
# save provider to user_input
user_input["provider"] = self.init_info["provider"]
Expand All @@ -373,7 +516,15 @@ async def async_step_aws_bedrock(self, user_input=None):
await aws_bedrock.validate()
# add the mode to user_input
user_input["provider"] = self.init_info["provider"]
return self.async_create_entry(title="AWS Bedrock", data=user_input)
if self.source == config_entries.SOURCE_RECONFIGURE:
# we're reconfiguring an existing config
return self.async_update_reload_and_abort(
self._get_reconfigure_entry(),
data_updates=user_input,
)
else:
# New config entry
return self.async_create_entry(title="AWS Bedrock Provider", data=user_input)
except ServiceValidationError as e:
_LOGGER.error(f"Validation failed: {e}")
return self.async_show_form(
Expand All @@ -386,3 +537,8 @@ async def async_step_aws_bedrock(self, user_input=None):
step_id="aws_bedrock",
data_schema=data_schema,
)

async def async_step_reconfigure(self, user_input):
data = self._get_reconfigure_entry().data
provider = data["provider"]
return await self.handle_provider(provider)
Loading

0 comments on commit dc9bd34

Please sign in to comment.