uezo / aiavatarkit

🥰 Building AI-based conversational avatars lightning fast ⚡️💬
Apache License 2.0
257 stars 21 forks source link

Add support for Claude function calling and vision input #53

Closed uezo closed 5 months ago

uezo commented 5 months ago

Function Calling

from aiavatar.processors.claude import ClaudeProcessor, ClaudeFunction

# Make function
weather_func_claude = ClaudeFunction(
    name="get_current_weather",
    description="Get the current weather in a given location",
    input_schema={
        "type": "object",
        "properties": {"location": {"type": "string", "description": "The city name of the location for which to get the weather."}},
    },
    func=weather
)

# Create ClaudeProcessor with functions
chat_processor = ClaudeProcessor(
    api_key=YOUR_API_KEY,
    functions={"get_current_weather": weather_func_claude}
)

Vision input

from aiavatar.processors.claude import ClaudeProcessorWithVisionBase

# Create ChatProcessor that captures screen shot
class ClaudeProcessorWithVisionScreenShot(ClaudeProcessorWithVisionBase):
    async def get_image(self) -> bytes:
        buffered = io.BytesIO()
        image = pyautogui.screenshot(region=(0, 0, 1280, 720))
        image.save(buffered, format="PNG")
        image.save("image_to_claude.png")   # Save current image for debug
        return buffered.getvalue()

chat_processor_claude = ClaudeProcessorWithVisionScreenShot(
    api_key=YOUR_API_KEY
)