search

langchain-ai / langgraph

Build resilient language agents as graphs.
https://langchain-ai.github.io/langgraph/
MIT License
4.24k stars 638 forks source link

llm.with_structured_output(RouteQuery) fails running AdaptiveRAG Example with AzureOpenAI #312

Open pratikkotian04 opened 2 months ago

pratikkotian04 commented 2 months ago

Checked other resources

Example Code

from langchain_openai import AzureChatOpenAI from langchain_openai import AzureOpenAIEmbeddings from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.document_loaders import WebBaseLoader from langchain_community.vectorstores import Chroma from typing import Literal from langchain_core.prompts import ChatPromptTemplate from langchain_core.pydantic_v1 import BaseModel, Field from langchain import hub from langchain_core.output_parsers import StrOutputParser from langchain_community.tools.tavily_search import TavilySearchResults

Docs to index

urls = [ "https://lilianweng.github.io/posts/2023-06-23-agent/", "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/", "https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/", ]

Load

docs = [WebBaseLoader(url).load() for url in urls] docs_list = [item for sublist in docs for item in sublist]

Split

text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( chunk_size=500, chunk_overlap=0 ) doc_splits = text_splitter.split_documents(docs_list)

Add to vectorstore

vectorstore = Chroma.from_documents( documents=doc_splits, collection_name="rag-chroma", embedding=embeddings, ) retriever = vectorstore.as_retriever()

Data model

class RouteQuery(BaseModel): """Route a user query to the most relevant datasource."""

datasource: Literal["vectorstore", "web_search"] = Field(
    ...,
    description="Given a user question choose to route it to web search or a vectorstore.",
)

LLM with function call

llm = AzureChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0) structured_llm_router = llm.with_structured_output(RouteQuery)

Prompt

system = """You are an expert at routing a user question to a vectorstore or web search. The vectorstore contains documents related to agents, prompt engineering, and adversarial attacks. Use the vectorstore for questions on these topics. Otherwise, use web-search.""" route_prompt = ChatPromptTemplate.from_messages( [ ("system", system), ("human", "{question}"), ] )

question_router = route_prompt | structured_llm_router print(question_router.invoke({"question": "Who will the Bears draft first in the NFL draft?"})) print(question_router.invoke({"question": "What are the types of agent memory?"}))

Error Message and Stack Trace (if applicable)

C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core_api\beta_decorator.py:87: LangChainBetaWarning: The function with_structured_output is in beta. It is actively being worked on, so the API may change. warn_beta( ---ROUTE QUESTION--- Traceback (most recent call last): File "C:\Users\prakotian\Desktop\Projects\GenAI Projects\AdaptiveRAG\app.py", line 255, in for output in app.stream(inputs): File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langgraph\pregel__init__.py", line 686, in stream _panic_or_proceed(done, inflight, step) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langgraph\pregel__init__.py", line 1049, in _panic_or_proceed raise exc File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\concurrent\futures\thread.py", line 58, in run result = self.fn(self.args, self.kwargs) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\base.py", line 2499, in invoke input = step.invoke( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langgraph\utils.py", line 49, in invoke ret = self.func(input, merge_configs(self.config, config), self.kwargs) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langgraph\graph\graph.py", line 67, in _route result = self.condition.invoke(reader(config) if reader else input, config) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\base.py", line 3961, in invoke return self._call_with_config( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\base.py", line 1625, in _call_with_config context.run( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\config.py", line 347, in call_func_with_variable_args return func(input, kwargs) # type: ignore[call-arg] File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\base.py", line 3835, in _invoke output = call_func_with_variable_args( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\config.py", line 347, in call_func_with_variable_args return func(input, kwargs) # type: ignore[call-arg] File "C:\Users\prakotian\Desktop\Projects\GenAI Projects\AdaptiveRAG\app.py", line 142, in route_question source = question_router.invoke({"question": question}) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\base.py", line 2499, in invoke input = step.invoke( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\base.py", line 4511, in invoke return self.bound.invoke( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\language_models\chat_models.py", line 158, in invoke self.generate_prompt( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\language_models\chat_models.py", line 560, in generate_prompt return self.generate(prompt_messages, stop=stop, callbacks=callbacks, kwargs) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\language_models\chat_models.py", line 421, in generate raise e File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\language_models\chat_models.py", line 411, in generate self._generate_with_cache( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\language_models\chat_models.py", line 632, in _generate_with_cache result = self._generate( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_openai\chat_models\base.py", line 548, in _generate response = self.client.create(messages=message_dicts, params) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\openai_utils_utils.py", line 275, in wrapper return func(args, **kwargs) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\openai\resources\chat\completions.py", line 667, in create return self._post( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\openai_base_client.py", line 1233, in post return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\openai_base_client.py", line 922, in request return self._request( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\openai_base_client.py", line 1013, in _request raise self._make_status_error_from_response(err.response) from None openai.BadRequestError: Error code: 400 - {'error': {'message': "Invalid parameter: 'response_format' of type 'json_object' is not supported with this model.", 'type': 'invalid_request_error', 'param': 'response_format', 'code': None}}

(Py10) C:\Users\prakotian\Desktop\Projects\GenAI Projects\AdaptiveRAG>

(Py10) C:\Users\prakotian\Desktop\Projects\GenAI Projects\AdaptiveRAG>python router.py C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core_api\beta_decorator.py:87: LangChainBetaWarning: The function with_structured_output is in beta. It is actively being worked on, so the API may change. warn_beta( Traceback (most recent call last): File "C:\Users\prakotian\Desktop\Projects\GenAI Projects\AdaptiveRAG\router.py", line 87, in print(question_router.invoke({"question": "Who will the Bears draft first in the NFL draft?"})) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\base.py", line 2499, in invoke input = step.invoke( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\output_parsers\base.py", line 169, in invoke return self._call_with_config( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\base.py", line 1625, in _call_with_config context.run( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\runnables\config.py", line 347, in call_func_with_variable_args return func(input, **kwargs) # type: ignore[call-arg] File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\output_parsers\base.py", line 170, in lambda inner_input: self.parse_result( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\output_parsers\openai_tools.py", line 182, in parse_result json_results = super().parse_result(result, partial=partial) File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\output_parsers\openai_tools.py", line 129, in parse_result tool_calls = parse_tool_calls( File "C:\Users\prakotian\AppData\Local\miniconda3\envs\Py10\lib\site-packages\langchain_core\output_parsers\openai_tools.py", line 85, in parse_tool_calls raise OutputParserException("\n\n".join(exceptions)) langchain_core.exceptions.OutputParserException: Function RouteQuery arguments:

{ datasource: "web_search" }

are not valid JSON. Received JSONDecodeError Expecting property name enclosed in double quotes: line 2 column 3 (char 4)

Description

Expected Output:-

datasource='web_search' datasource='vectorstore'

System Info

System Information

OS: Windows OS Version: 10.0.19045 Python Version: 3.10.13 | packaged by Anaconda, Inc. | (main, Sep 11 2023, 13:24:38) [MSC v.1916 64 bit (AMD64)]

Package Information

langchain_core: 0.1.43 langchain: 0.1.16 langchain_community: 0.0.33 langsmith: 0.1.31 langchain_cohere: 0.1.2 langchain_experimental: 0.0.54 langchain_openai: 0.1.3 langchain_text_splitters: 0.0.1 langchainhub: 0.1.15 langgraph: 0.0.37

ebalp commented 2 months ago

I am getting a very similar error

KeyanatGiggso commented 2 months ago

Yes, I'm also

BraveStone9 commented 1 month ago

I am facing a similar issue. Although my LLM with function call slightly differs in client call.

llm = AzureChatOpenAI( openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"], azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], )

structured_llm_grader = llm.with_structured_output(GradeDocuments, method="json_mode")

This gives a similar error. And If I remove json_mode, the following error:

GradeDocuments({binary_score: "no"})

are not valid JSON. Received JSONDecodeError Expecting value: line 1 column 1 (char 0)

lzytitan494 commented 1 week ago

I was also having the same issue with using Cohere's command-r-plus. But instead of using .with_structured_output() I changed to `.bindtools()'.

My code:

`

Router

from langchain_core.prompts import ChatPromptTemplate, FewShotChatMessagePromptTemplate from langchain_core.pydantic_v1 import BaseModel, Field from langchain_cohere import ChatCohere

Data model

class web_search(BaseModel): """ The internet. Use web_search for questions that are related to anything else than agents, prompt engineering, and adversarial attacks. """

query: str = Field(description="The query to use when searching the internet.")

class vectorstore(BaseModel): """ A vectorstore containing documents related to Graph Neural Networks. Use the vectorstore for questions on these topics. """

query: str = Field(description="The query to use when searching the vectorstore.")

LLM with function call

llm = ChatCohere(model="command-r-plus", temperature=0) structured_llm_router = llm.bind_tools( tools=[web_search, vectorstore] )

Few shot prompting

routing_examples = [ {"question": "What are the main components of a Graph Neural Network?", "decision": "vectorstore"}, {"question": "Who won the Nobel Prize in Physics in 2023?", "decision": "web_search"}, {"question": "How do GNNs handle the over-smoothing problem?", "decision": "vectorstore"}, {"question": "What is the capital of Japan?", "decision": "web_search"}, {"question": "Can you explain how Graph Attention Networks (GATs) work?", "decision": "vectorstore"}, ]

example_prompt = ChatPromptTemplate.from_messages( [ ("human", "{question}"), ("ai", "{decision}"), ] )

few_shot_prompt = FewShotChatMessagePromptTemplate( example_prompt=example_prompt, examples=routing_examples, )

Prompt

system = """You are an expert at routing a user question to a vectorstore or web search. The vectorstore contains documents related to Graph Neural Networks (GNNs). Use the vectorstore for questions on these topics. Otherwise, use web-search.""" route_prompt = ChatPromptTemplate.from_messages( [ ("system", system), few_shot_prompt, ("human", "{question}"), ] )

question_router = route_prompt | structured_llm_router

response = question_router.invoke({"question": "Who won the Nobel Prize in Physics in 2023?"}) print(response.response_metadata["tool_calls"]) response = question_router.invoke({"question": "What are the types of Graph Neural Network?"}) print(response.response_metadata["tool_calls"]) response = question_router.invoke({"question": "Hi how are you?"}) # answers directly print("tool_calls" in response.response_metadata) `

Response: [{'id': '30415ef1fb324ca28ad456109271c5f4', 'function': {'name': 'web_search', 'arguments': '{"query": "Nobel Prize Physics 2023"}'}, 'type': 'function'}] [{'id': '992dc30308e44e328ad1155606ad6038', 'function': {'name': 'vectorstore', 'arguments': '{"query": "types of graph neural network"}'}, 'type': 'function'}] False