Zod Error when using image input with "pixtral-12b-2409" model

[6-2-1-5]

Description

When attempting to use the Mistral AI library to send an image input to the "pixtral-12b-2409" model, I'm encountering a Zod validation error. The error suggests a mismatch between the data structure I'm providing and what the schema expects.

Code

I just copied and pasted it directly from the official docs here

const apiKey = process.env.MISTRAL_API_KEY;

const mistral = new Mistral({apiKey: apiKey});

const chatResponse = await mistral.chat.complete({
    model: "pixtral-12b-2409",
    messages: [
        {
        "role": "user",
        "content": [
            {
            "type": "text",
            "text": "What’s in this image?"
            },
            {
            "type": "image_url",
            "image_url": "https://tripfixers.com/wp-content/uploads/2019/11/eiffel-tower-with-snow.jpeg"
            }
        ]
        }
    ]
    }
);

console.log('JSON:', chatResponse.choices[0].message.content)

Error Message

ZodError: [ { "code": "invalid_union", "unionErrors": [ { "issues": [ { "code": "invalid_union", "unionErrors": [ { "issues": [ { "code": "invalid_type", "expected": "string", "received": "array", "path": [ "messages", 0, "content" ], "message": "Expected string, received array" } ], "name": "ZodError" }, { "issues": [ { "received": "image_url", "code": "invalid_literal", "expected": "text", "path": [ "messages", 0, "content", 1, "type" ], "message": "Invalid literal value, expected \"text\"" }, { "code": "invalid_type", "expected": "string", "received": "undefined", "path": [ "messages", 0, "content", 1, "text" ], "message": "Required" } ], "name": "ZodError" } ], "path": [ "messages", 0, "content" ], "message": "Invalid input" }, { "received": "user", "code": "invalid_enum_value", "options": [ "system" ], "path": [ "messages", 0, "role" ], "message": "Invalid enum value. Expected 'system', received 'user'" }, { "received": "user", "code": "invalid_literal", "expected": "system", "path": [ "messages", 0, "role" ], "message": "Invalid literal value, expected \"system\"" } ], "name": "ZodError" }, { "issues": [ { "code": "invalid_union", "unionErrors": [ { "issues": [ { "code": "invalid_type", "expected": "string", "received": "array", "path": [ "messages", 0, "content" ], "message": "Expected string, received array" } ], "name": "ZodError" }, { "issues": [ { "code": "invalid_union", "unionErrors": [ { "issues": [ { "code": "invalid_union", "unionErrors": [ { "issues": [ { "code": "invalid_type", "expected": "object", "received": "undefined", "path": [ "messages", 0, "content", 1, "imageUrl" ], "message": "Required" } ], "name": "ZodError" }, { "issues": [ { "code": "invalid_type", "expected": "string", "received": "undefined", "path": [ "messages", 0, "content", 1, "imageUrl" ], "message": "Required" } ], "name": "ZodError" } ], "path": [ "messages", 0, "content", 1, "imageUrl" ], "message": "Invalid input" } ], "name": "ZodError" }, { "issues": [ { "received": "image_url", "code": "invalid_literal", "expected": "text", "path": [ "messages", 0, "content", 1, "type" ], "message": "Invalid literal value, expected \"text\"" }, { "code": "invalid_type", "expected": "string", "received": "undefined", "path": [ "messages", 0, "content", 1, "text" ], "message": "Required" }, { "received": "image_url", "code": "invalid_literal", "expected": "text", "path": [ "messages", 0, "content", 1, "type" ], "message": "Invalid literal value, expected \"text\"" } ], "name": "ZodError" } ], "path": [ "messages", 0, "content", 1 ], "message": "Invalid input" } ], "name": "ZodError" } ], "path": [ "messages", 0, "content" ], "message": "Invalid input" } ], "name": "ZodError" }, { "issues": [ { "code": "invalid_type", "expected": "string", "received": "array", "path": [ "messages", 0, "content" ], "message": "Expected string, received array" }, { "received": "user", "code": "invalid_enum_value", "options": [ "assistant" ], "path": [ "messages", 0, "role" ], "message": "Invalid enum value. Expected 'assistant', received 'user'" }, { "received": "user", "code": "invalid_literal", "expected": "assistant", "path": [ "messages", 0, "role" ], "message": "Invalid literal value, expected \"assistant\"" } ], "name": "ZodError" }, { "issues": [ { "code": "invalid_type", "expected": "string", "received": "array", "path": [ "messages", 0, "content" ], "message": "Expected string, received array" }, { "received": "user", "code": "invalid_enum_value", "options": [ "tool" ], "path": [ "messages", 0, "role" ], "message": "Invalid enum value. Expected 'tool', received 'user'" }, { "received": "user", "code": "invalid_literal", "expected": "tool", "path": [ "messages", 0, "role" ], "message": "Invalid literal value, expected \"tool\"" } ], "name": "ZodError" } ], "path": [ "messages", 0 ], "message": "Invalid input" } ] errors: [ { "code": "invalid_union", "unionErrors": [ { "issues": [ { "code": "invalid_union", "unionErrors": [ { "issues": [ { "code": "invalid_type", "expected": "string", "received": "array", "path": [ "messages", 0, "content" ], "message": "Expected string, received array" } ], "name": "ZodError" }, { "issues": [ { "received": "image_url", "code": "invalid_literal", "expected": "text", "path": [ "messages", 0, "content", 1, "type" ], "message": "Invalid literal value, expected \"text\"" }, { "code": "invalid_type", "expected": "string", "received": "undefined", "path": [ "messages", 0, "content", 1, "text" ], "message": "Required" } ], "name": "ZodError" } ], "path": [ "messages", 0, "content" ], "message": "Invalid input" }, { "received": "user", "code": "invalid_enum_value", "options": [ "system" ], "path": [ "messages", 0, "role" ], "message": "Invalid enum value. Expected 'system', received 'user'" }, { "received": "user", "code": "invalid_literal", "expected": "system", "path": [ "messages", 0, "role" ], "message": "Invalid literal value, expected \"system\"" } ], "name": "ZodError" }, { "issues": [ { "code": "invalid_union", "unionErrors": [ { "issues": [ { "code": "invalid_type", "expected": "string", "received": "array", "path": [ "messages", 0, "content" ], "message": "Expected string, received array" } ], "name": "ZodError" }, { "issues": [ { "code": "invalid_union", "unionErrors": [ { "issues": [ { "code": "invalid_union", "unionErrors": [ { "issues": [ { "code": "invalid_type", "expected": "object", "received": "undefined", "path": [ "messages", 0, "content", 1, "imageUrl" ], "message": "Required" } ], "name": "ZodError" }, { "issues": [ { "code": "invalid_type", "expected": "string", "received": "undefined", "path": [ "messages", 0, "content", 1, "imageUrl" ], "message": "Required" } ], "name": "ZodError" } ], "path": [ "messages", 0, "content", 1, "imageUrl" ], "message": "Invalid input" } ], "name": "ZodError" }, { "issues": [ { "received": "image_url", "code": "invalid_literal", "expected": "text", "path": [ "messages", 0, "content", 1, "type" ], "message": "Invalid literal value, expected \"text\"" }, { "code": "invalid_type", "expected": "string", "received": "undefined", "path": [ "messages", 0, "content", 1, "text" ], "message": "Required" }, { "received": "image_url", "code": "invalid_literal", "expected": "text", "path": [ "messages", 0, "content", 1, "type" ], "message": "Invalid literal value, expected \"text\"" } ], "name": "ZodError" } ], "path": [ "messages", 0, "content", 1 ], "message": "Invalid input" } ], "name": "ZodError" } ], "path": [ "messages", 0, "content" ], "message": "Invalid input" } ], "name": "ZodError" }, { "issues": [ { "code": "invalid_type", "expected": "string", "received": "array", "path": [ "messages", 0, "content" ], "message": "Expected string, received array" }, { "received": "user", "code": "invalid_enum_value", "options": [ "assistant" ], "path": [ "messages", 0, "role" ], "message": "Invalid enum value. Expected 'assistant', received 'user'" }, { "received": "user", "code": "invalid_literal", "expected": "assistant", "path": [ "messages", 0, "role" ], "message": "Invalid literal value, expected \"assistant\"" } ], "name": "ZodError" }, { "issues": [ { "code": "invalid_type", "expected": "string", "received": "array", "path": [ "messages", 0, "content" ], "message": "Expected string, received array" }, { "received": "user", "code": "invalid_enum_value", "options": [ "tool" ], "path": [ "messages", 0, "role" ], "message": "Invalid enum value. Expected 'tool', received 'user'" }, { "received": "user", "code": "invalid_literal", "expected": "tool", "path": [ "messages", 0, "role" ], "message": "Invalid literal value, expected \"tool\"" } ], "name": "ZodError" } ], "path": [ "messages", 0 ], "message": "Invalid input" } ]

[GaspardBT]

Thanks for reporting the issue, we will investigate the issue. In the meantime, you can use the following syntax:

const chatResponse = await client.chat.complete({
  model: "pixtral-12b",
  messages: [
    {
      role: "user",
      content: [
        { type: "text", text: "What's in this image?" },
        {
          type: "image_url",
          imageUrl: "https://mistral.ai/images/news/codestral/FIM_table.png",
        },
      ]
    }
  ],
});

[6-2-1-5]

Thanks for reporting the issue, we will investigate the issue. In the meantime, you can use the following syntax:

const chatResponse = await client.chat.complete({
  model: "pixtral-12b",
  messages: [
    {
      role: "user",
      content: [
        { type: "text", text: "What's in this image?" },
        {
          type: "image_url",
          imageUrl: "https://mistral.ai/images/news/codestral/FIM_table.png",
        },
      ]
    }
  ],
});

Thanks!

[GaspardBT]

After some investigation, to make it works with the original syntax you should use:

 {
  "type": "image_url",
  "imageUrl": "https://tripfixers.com/wp-content/uploads/2019/11/eiffel-tower-with-snow.jpeg"
 }

image_url -> imageUrl We are updating the doc to use the syntax I shared with you in https://github.com/mistralai/platform-docs-public/pull/138 and we added an example as well in https://github.com/mistralai/client-ts/pull/31

Let us know if you have more questions 🚀