search

langchain-ai / langchainjs

🦜🔗 Build context-aware reasoning applications 🦜🔗
https://js.langchain.com/docs/
MIT License
12.29k stars 2.08k forks source link

JSON schema was not passed on to the OpenAI request when using ReAct agent #6623

Open bhaveshpatel640 opened 2 weeks ago

bhaveshpatel640 commented 2 weeks ago

Checked other resources

Example Code

import { ChatOpenAI } from "@langchain/openai";
import { DynamicStructuredTool } from "@langchain/core/tools";
import { createReactAgent } from "@langchain/langgraph/prebuilt";
import { BaseChatModel } from "@langchain/core/language_models/chat_models";
import { MemorySaver } from "@langchain/langgraph";
import { z } from "zod";
import { v4 } from "uuid";

const func = async (input: any): Promise<string> => {
  console.log("input", input); // <= When schema is json schema (USE_TOOL_SCHEMA: true), input is {} always
  if (input && input.breed) {
    return JSON.stringify({
      message: "Here are the pets",
      pets: ["pet a", "pet b", "pet c"],
    });
  } else {
    return JSON.stringify({
      message: "Please provide a breed",
      pets: [],
    });
  }
};

let schema;
const USE_TOOL_SCHEMA = true;  // <= Change this to false to use zod schema

if (USE_TOOL_SCHEMA) {
  schema = {
    type: "object",
    additionalProperties: false,
    required: ["breed"],
    properties: {
      breed: {
        type: "string",
        description: "Breed of the pet you are looking for",
      },
    },
    $schema: "http://json-schema.org/draft-07/schema#",
  };
} else {
  schema = z.object({
    breed: z.string().describe("Breed of the pet you are looking for"),
  });
}

const petStoreTool = new DynamicStructuredTool({
  name: "pet_store",
  description: "Get a list of pets based on breed from the pet store",
  schema,
  func,
});

const instruction = `You are a knowledgeable assistant at a pet store.
  Your task is to help users find pets based on their specific breed queries.
  When a user provides a breed, you should respond with a list of available pets that match that breed or inform them if no such pets are available.
  Be clear, concise, and ensure your responses are relevant to the breed mentioned.
  DO not assume any additional information about the user's query.`;

const llm = new ChatOpenAI({
  temperature: 0,
  model: "gpt-3.5-turbo",
  configuration: {
    fetch: async (url: RequestInfo, init?: RequestInit): Promise<Response> => {
      // console.log("About to make a request", url, init);
      const response = await fetch(url, init);
      return response;
    },
  },
});

const agentExecutor = createReactAgent({
  tools: [petStoreTool],
  llm: llm as unknown as BaseChatModel,
  messageModifier: instruction,
  checkpointSaver: new MemorySaver(),
});

const input =
  "I'm looking for a pet golden dog cross breed. Can you help me find one?";

async function run() {
  const langChainStream = await agentExecutor.stream(
    { input: input },
    {
      streamMode: "updates",
      configurable: { thread_id: v4() },
    }
  );
  for await (const step of langChainStream) {
    console.log(JSON.stringify(step));
  }
}

// Execute the function
run().catch(console.error);

Error Message and Stack Trace (if applicable)

No response

Description

// When USE_TOOL_SCHEMA is true

user@system agent_langchain % npx ts-node app.ts

{"agent":{"messages":[{"lc":1,"type":"constructor","id":["langchain_core","messages","AIMessage"],"kwargs":{"content":"","tool_calls":[{"name":"pet_store","args":{},"type":"tool_call","id":"call_id_1"}],"invalid_tool_calls":[],"additional_kwargs":{"tool_calls":[{"id":"call_id_1","type":"function","function":{"name":"pet_store","arguments":"{}"}}]},"response_metadata":{"tokenUsage":{"completionTokens":10,"promptTokens":128,"totalTokens":138},"finish_reason":"tool_calls"},"id":"chatcmpl-1"}}]}}

input {}

{"tools":{"messages":[{"lc":1,"type":"constructor","id":["langchain_core","messages","ToolMessage"],"kwargs":{"content":"{\"message\":\"Please provide a breed\",\"pets\":[]}","tool_call_id":"call_id_1","name":"pet_store","additional_kwargs":{},"response_metadata":{}}}]}}
{"agent":{"messages":[{"lc":1,"type":"constructor","id":["langchain_core","messages","AIMessage"],"kwargs":{"content":"I'm sorry, but I couldn't find any pets matching the breed you mentioned. Please provide a different breed for me to search for available pets.","tool_calls":[],"invalid_tool_calls":[],"additional_kwargs":{},"response_metadata":{"tokenUsage":{"completionTokens":31,"promptTokens":157,"totalTokens":188},"finish_reason":"stop"},"id":"chatcmpl-1"}}]}}

// When USE_TOOL_SCHEMA is False

user@system agent_langchain % npx ts-node app.ts

{"agent":{"messages":[{"lc":1,"type":"constructor","id":["langchain_core","messages","AIMessage"],"kwargs":{"content":"","tool_calls":[{"name":"pet_store","args":{"breed":"Labrador Retriever"},"type":"tool_call","id":"call_sUjEMxqbwPWtPmJdx3mjMbz2"}],"invalid_tool_calls":[],"additional_kwargs":{"tool_calls":[{"id":"call_sUjEMxqbwPWtPmJdx3mjMbz2","type":"function","function":{"name":"pet_store","arguments":"{\"breed\":\"Labrador Retriever\"}"}}]},"response_metadata":{"tokenUsage":{"completionTokens":19,"promptTokens":146,"totalTokens":165},"finish_reason":"tool_calls"},"id":"chatcmpl-1"}}]}}

input { breed: 'Labrador Retriever' }

{"tools":{"messages":[{"lc":1,"type":"constructor","id":["langchain_core","messages","ToolMessage"],"kwargs":{"content":"{\"message\":\"Here are the pets\",\"pets\":[\"pet a\",\"pet b\",\"pet c\"]}","tool_call_id":"call_sUjEMxqbwPWtPmJdx3mjMbz2","name":"pet_store","additional_kwargs":{},"response_metadata":{}}}]}}
{"agent":{"messages":[{"lc":1,"type":"constructor","id":["langchain_core","messages","AIMessage"],"kwargs":{"content":"We have the following pets available that match the Labrador Retriever breed: pet a, pet b, and pet c.","tool_calls":[],"invalid_tool_calls":[],"additional_kwargs":{},"response_metadata":{"tokenUsage":{"completionTokens":26,"promptTokens":192,"totalTokens":218},"finish_reason":"stop"},"id":"chatcmpl-1"}}]}}

System Info

node --version

v20.15.1

yarn --version

1.22.22
{
  name: 'langchain',
  version: '0.2.17',
  type: 'module',
  engines: {
    node: '>=18'
  },
  'dist-tags': {
    next: '0.2.3-rc.0',
    latest: '0.2.17'
  },
  ....
}
dosubot[bot] commented 2 weeks ago

Hey @bhaveshpatel640! I'm here to help you with any bugs, questions, or contributions you have in mind. Let's dive in and tackle this together!

I found a similar issue where the JSON schema was not passed on to the OpenAI request when using the ReAct agent in LangChain.js. The root cause is that the OpenAIInput and related interfaces do not include a field for the JSON schema. The OpenAIInput interface and its related types define various parameters for making requests to OpenAI, but there is no specific field for including a JSON schema in the request [1][2].

To work around this, you can use the JsonToolkit and JsonSpec classes to create agents that understand and utilize JSON schemas. Here is an example of how to create such an agent:

import * as fs from "fs";
import * as yaml from "js-yaml";
import { OpenAI } from "@langchain/openai";
import { JsonSpec, JsonObject } from "langchain/tools";
import { JsonToolkit, createJsonAgent } from "langchain/agents";

export const run = async () => {
  let data: JsonObject;
  try {
    const yamlFile = fs.readFileSync("openai_openapi.yaml", "utf8");
    data = yaml.load(yamlFile) as JsonObject;
    if (!data) {
      throw new Error("Failed to load OpenAPI spec");
    }
  } catch (e) {
    console.error(e);
    return;
  }

  const toolkit = new JsonToolkit(new JsonSpec(data));
  const model = new OpenAI({ temperature: 0 });
  const executor = createJsonAgent(model, toolkit);

  const input = `What are the required parameters in the request body to the /completions endpoint?`;

  console.log(`Executing with input "${input}"...`);

  const result = await executor.invoke({ input });

  console.log(`Got output ${result.output}`);

  console.log(
    `Got intermediate steps ${JSON.stringify(
      result.intermediateSteps,
      null,
      2
    )}`
  );
};

This approach allows for the use of JSON schemas in a structured manner, even though the schema itself is not directly passed in the OpenAI request [3].

Additionally, LangChain.js provides utilities to convert Zod schemas to JSON schemas and use them in tagging chains. Here is an example of how to create a metadata tagger using a Zod schema:

import { z } from "zod";
import { createMetadataTaggerFromZod } from "langchain/src/document_transformers/openai_functions";

const schema = z.object({
  title: z.string(),
  author: z.string(),
});

const options = {
  llm: new ChatOpenAI({ modelName: "gpt-3.5-turbo-0613" }),
  // other options...
};

const metadataTagger = createMetadataTaggerFromZod(schema, options);

This demonstrates how LangChain.js can work with JSON schemas indirectly through Zod schemas and other utilities [4].

To continue talking to Dosu, mention @dosu.

TimSusa commented 2 weeks ago

[3]

OK, sounds cool to me at first, but then I see your code, while I see a yml file is read and loaded. I am sorry, but where to me I got a cold gosebumps on my neck, why that should be necessary?