Arize-ai / openinference

Auto-Instrumentation for AI Observability
https://arize-ai.github.io/openinference/
Apache License 2.0
146 stars 22 forks source link

feat: Add multimodal content arrays to llm messages #542

Closed fjcasti1 closed 3 days ago

fjcasti1 commented 6 days ago

Example span when running python examples/chat_completions_multimodal.py

{
    "name": "ChatCompletion",
    "context": {
        "trace_id": "0x84d562932c4b2baa836e4c9e63dfb406",
        "span_id": "0x8c8537700758be23",
        "trace_state": "[]"
    },
    "kind": "SpanKind.INTERNAL",
    "parent_id": null,
    "start_time": "2024-06-24T23:53:33.339233Z",
    "end_time": "2024-06-24T23:53:40.850425Z",
    "status": {
        "status_code": "OK"
    },
    "attributes": {
        "openinference.span.kind": "LLM",
        "input.value": "{\"messages\": [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"What is in this image?\"}, {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}}]}], \"model\": \"gpt-4o\", \"max_tokens\": 300}",
        "input.mime_type": "application/json",
        "output.value": "{\"id\":\"chatcmpl-9dnn92jL2swNaOUY6vPSavVVmr7mL\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"The image shows a scenic outdoor landscape featuring a wooden boardwalk path extending forward through a lush green field. The field is filled with tall grass and some shrubs. In the background, there are some trees and bushes, and the sky above is clear with a few scattered clouds, giving a sense of a calm and pleasant day.\",\"role\":\"assistant\"}}],\"created\":1719273219,\"model\":\"gpt-4o-2024-05-13\",\"object\":\"chat.completion\",\"system_fingerprint\":\"fp_9cb5d38cf7\",\"usage\":{\"completion_tokens\":65,\"prompt_tokens\":1118,\"total_tokens\":1183}}",
        "output.mime_type": "application/json",
        "session.id": "my-test-session",
        "user.id": "my-test-user",
        "metadata": "{\"test-int\": 1, \"test-str\": \"string\", \"test-list\": [1, 2, 3], \"test-dict\": {\"key-1\": \"val-1\", \"key-2\": \"val-2\"}}",
        "tag.tags": [
            "tag-1",
            "tag-2"
        ],
        "llm.prompt_template.template": "Who won the soccer match in {city} on {date}",
        "llm.prompt_template.version": "v1.0",
        "llm.prompt_template.variables": "{\"city\": \"Johannesburg\", \"date\": \"July 11th\"}",
        "llm.invocation_parameters": "{\"model\": \"gpt-4o\", \"max_tokens\": 300}",
        "llm.input_messages.0.message.role": "user",
        "llm.input_messages.0.message.contents.1.message_content.type": "image",
        "llm.input_messages.0.message.contents.1.message_content.image.image.url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
        "llm.input_messages.0.message.contents.0.message_content.type": "text",
        "llm.input_messages.0.message.contents.0.message_content.text": "What is in this image?",
        "llm.model_name": "gpt-4o-2024-05-13",
        "llm.token_count.total": 1183,
        "llm.token_count.prompt": 1118,
        "llm.token_count.completion": 65,
        "llm.output_messages.0.message.role": "assistant",
        "llm.output_messages.0.message.content": "The image shows a scenic outdoor landscape featuring a wooden boardwalk path extending forward through a lush green field. The field is filled with tall grass and some shrubs. In the background, there are some trees and bushes, and the sky above is clear with a few scattered clouds, giving a sense of a calm and pleasant day."
    },
    "events": [],
    "links": [],
    "resource": {
        "attributes": {
            "telemetry.sdk.language": "python",
            "telemetry.sdk.name": "opentelemetry",
            "telemetry.sdk.version": "1.24.0",
            "service.name": "unknown_service"
        },
        "schema_url": ""
    }
}