dosubot[bot]
commented
4 months ago The responseMimeType configuration option works with ChatVertexAI but not with ChatGoogleGenerativeAI because the ChatGoogleGenerativeAI implementation does not include handling for the responseMimeType option.
To resolve this inconsistency, you need to add support for the responseMimeType option in the ChatGoogleGenerativeAI implementation. This involves modifying the convertMessageContentToParts function to handle the responseMimeType configuration.
Here is an example of how you can modify the convertMessageContentToParts function to include the responseMimeType option:
function messageContentMedia(content: MessageContentComplex): Part {
if ("mimeType" in content && "data" in content) {
return {
inlineData: {
mimeType: content.mimeType,
data: content.data,
},
};
}
throw new Error("Invalid media content");
}
export function convertMessageContentToParts(
message: BaseMessage,
isMultimodalModel: boolean
): Part[] {
if (typeof message.content === "string" && message.content !== "") {
return [{ text: message.content }];
}
let functionCalls: FunctionCallPart[] = [];
let functionResponses: FunctionResponsePart[] = [];
let messageParts: Part[] = [];
if (
"tool_calls" in message &&
Array.isArray(message.tool_calls) &&
message.tool_calls.length > 0
) {
functionCalls = message.tool_calls.map((tc) => ({
functionCall: {
name: tc.name,
args: tc.args,
},
}));
} else if (message._getType() === "tool" && message.name && message.content) {
functionResponses = [
{
functionResponse: {
name: message.name,
response: message.content,
},
},
];
} else if (Array.isArray(message.content)) {
messageParts = message.content.map((c) => {
if (c.type === "text") {
return {
text: c.text,
};
}
if (c.type === "image_url") {
if (!isMultimodalModel) {
throw new Error(`This model does not support images`);
}
let source;
if (typeof c.image_url === "string") {
source = c.image_url;
} else if (typeof c.image_url === "object" && "url" in c.image_url) {
source = c.image_url.url;
} else {
throw new Error("Please provide image as base64 encoded data URL");
}
const [dm, data] = source.split(",");
if (!dm.startsWith("data:")) {
throw new Error("Please provide image as base64 encoded data URL");
}
const [mimeType, encoding] = dm.replace(/^data:/, "").split(";");
if (encoding !== "base64") {
throw new Error("Please provide image as base64 encoded data URL");
}
return {
inlineData: {
data,
mimeType,
},
};
} else if (c.type === "media") {
return messageContentMedia(c);
} else if (c.type === "tool_use") {
return {
functionCall: {
name: c.name,
args: c.input,
},
};
}
throw new Error(`Unknown content type ${(c as { type: string }).type}`);
});
}
return [...messageParts, ...functionCalls, ...functionResponses];
}By adding the responseMimeType property handling in the convertMessageContentToParts function, you ensure that the responseMimeType option is properly handled in the ChatGoogleGenerativeAI implementation [1][2].
To continue talking to Dosu, mention @dosu.
afirstenberg
rossanodr
Checked other resources
Example Code
import { ChatVertexAI, ChatGoogleGenerativeAI } from "@langchain/google-genai";
const vertexAIChat = new ChatVertexAI({ responseMimeType: "application/json" // or "text/plain" });
const googleGenerativeAIChat = new ChatGoogleGenerativeAI({ responseMimeType: "application/json" }); // This results in the error: // Object literal may only specify known properties, and 'responseMimeType' does not exist in type 'GoogleGenerativeAIChatInput'.ts(2353)
Error Message and Stack Trace (if applicable)
No response
Description
I'm trying to use the responseMimeType configuration option with the new ChatVertexAI class, and it works as expected. However, when I try to use the same option with the new ChatGoogleGenerativeAI class, it results in the error:
Object literal may only specify known properties, and 'responseMimeType' does not exist in type 'GoogleGenerativeAIChatInput'.ts(2353)System Info
yarn info langchain └─ langchain@npm:0.2.5 ├─ Instances: 1 ├─ Version: 0.2.5 │ └─ Dependencies ├─ @langchain/core@npm:^0.2.5 → npm:0.2.7 ├─ @langchain/openai@npm:~0.1.0 → npm:0.1.3 ├─ @langchain/textsplitters@npm:~0.0.0 → npm:0.0.3 ├─ binary-extensions@npm:^2.2.0 → npm:2.3.0 ├─ js-tiktoken@npm:^1.0.12 → npm:1.0.12 ├─ js-yaml@npm:^4.1.0 → npm:4.1.0 ├─ jsonpointer@npm:^5.0.1 → npm:5.0.1 ├─ langchainhub@npm:~0.0.8 → npm:0.0.11 ├─ langsmith@npm:~0.1.30 → npm:0.1.32 ├─ ml-distance@npm:^4.0.0 → npm:4.0.1 ├─ openapi-types@npm:^12.1.3 → npm:12.1.3 ├─ p-retry@npm:4 → npm:4.6.2 ├─ uuid@npm:^9.0.0 → npm:9.0.1 ├─ yaml@npm:^2.2.1 → npm:2.4.5 ├─ zod-to-json-schema@npm:^3.22.3 → npm:3.23.1 └─ zod@npm:^3.22.4 → npm:3.23.8
"@langchain/core": "^0.2.5", "@langchain/google-genai": "^0.0.21", "@langchain/google-vertexai": "^0.0.17", "@langchain/google-vertexai-web": "^0.0.17",