ChatWatsonx with Structured output is failing

Full stack trace:
{
    "name": "TypeError",
    "message": "ModelInference.generate() got an unexpected keyword argument 'response_format'",
    "stack": "---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[3], line 22
     12 llm = ChatWatsonx(
     13     model_id=\"ibm/granite-13b-chat-v2\",
     14     url=",
     15     project_id=,
     16     params={\"decoding_method\": \"greedy\", \"max_new_tokens\": 100},
     17 )
     18 structured_llm = llm.with_structured_output(
     19     AnswerWithJustification, method=\"json_mode\", include_raw=True
     20 )
---> 22 structured_llm.invoke(
     23     \"Answer the following question. \"
     24     \"Make sure to return a JSON blob with keys 'answer' and 'justification'.\
\
\"
     25     \"What's heavier a pound of bricks or a pound of feathers?\"
     26 )

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_core/runnables/base.py:3020, in RunnableSequence.invoke(self, input, config, **kwargs)
   3018 context.run(_set_config_context, config)
   3019 if i == 0:
-> 3020     input = context.run(step.invoke, input, config, **kwargs)
   3021 else:
   3022     input = context.run(step.invoke, input, config)

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_core/runnables/base.py:3723, in RunnableParallel.invoke(self, input, config)
   3718     with get_executor_for_config(config) as executor:
   3719         futures = [
   3720             executor.submit(_invoke_step, step, input, config, key)
   3721             for key, step in steps.items()
   3722         ]
-> 3723         output = {key: future.result() for key, future in zip(steps, futures)}
   3724 # finish the root run
   3725 except BaseException as e:

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_core/runnables/base.py:3723, in <dictcomp>(.0)
   3718     with get_executor_for_config(config) as executor:
   3719         futures = [
   3720             executor.submit(_invoke_step, step, input, config, key)
   3721             for key, step in steps.items()
   3722         ]
-> 3723         output = {key: future.result() for key, future in zip(steps, futures)}
   3724 # finish the root run
   3725 except BaseException as e:

File ~/.pyenv/versions/3.11.9/lib/python3.11/concurrent/futures/_base.py:449, in Future.result(self, timeout)
    447     raise CancelledError()
    448 elif self._state == FINISHED:
--> 449     return self.__get_result()
    451 self._condition.wait(timeout)
    453 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:

File ~/.pyenv/versions/3.11.9/lib/python3.11/concurrent/futures/_base.py:401, in Future.__get_result(self)
    399 if self._exception:
    400     try:
--> 401         raise self._exception
    402     finally:
    403         # Break a reference cycle with the exception in self._exception
    404         self = None

File ~/.pyenv/versions/3.11.9/lib/python3.11/concurrent/futures/thread.py:58, in _WorkItem.run(self)
     55     return
     57 try:
---> 58     result = self.fn(*self.args, **self.kwargs)
     59 except BaseException as exc:
     60     self.future.set_exception(exc)

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_core/runnables/base.py:3707, in RunnableParallel.invoke.<locals>._invoke_step(step, input, config, key)
   3705 context = copy_context()
   3706 context.run(_set_config_context, child_config)
-> 3707 return context.run(
   3708     step.invoke,
   3709     input,
   3710     child_config,
   3711 )

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_core/runnables/base.py:5343, in RunnableBindingBase.invoke(self, input, config, **kwargs)
   5337 def invoke(
   5338     self,
   5339     input: Input,
   5340     config: Optional[RunnableConfig] = None,
   5341     **kwargs: Optional[Any],
   5342 ) -> Output:
-> 5343     return self.bound.invoke(
   5344         input,
   5345         self._merge_configs(config),
   5346         **{**self.kwargs, **kwargs},
   5347     )

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_core/language_models/chat_models.py:284, in BaseChatModel.invoke(self, input, config, stop, **kwargs)
    273 def invoke(
    274     self,
    275     input: LanguageModelInput,
   (...)
    279     **kwargs: Any,
    280 ) -> BaseMessage:
    281     config = ensure_config(config)
    282     return cast(
    283         ChatGeneration,
--> 284         self.generate_prompt(
    285             [self._convert_input(input)],
    286             stop=stop,
    287             callbacks=config.get(\"callbacks\"),
    288             tags=config.get(\"tags\"),
    289             metadata=config.get(\"metadata\"),
    290             run_name=config.get(\"run_name\"),
    291             run_id=config.pop(\"run_id\", None),
    292             **kwargs,
    293         ).generations[0][0],
    294     ).message

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_core/language_models/chat_models.py:784, in BaseChatModel.generate_prompt(self, prompts, stop, callbacks, **kwargs)
    776 def generate_prompt(
    777     self,
    778     prompts: list[PromptValue],
   (...)
    781     **kwargs: Any,
    782 ) -> LLMResult:
    783     prompt_messages = [p.to_messages() for p in prompts]
--> 784     return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_core/language_models/chat_models.py:641, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    639         if run_managers:
    640             run_managers[i].on_llm_error(e, response=LLMResult(generations=[]))
--> 641         raise e
    642 flattened_outputs = [
    643     LLMResult(generations=[res.generations], llm_output=res.llm_output)  # type: ignore[list-item]
    644     for res in results
    645 ]
    646 llm_output = self._combine_llm_outputs([res.llm_output for res in results])

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_core/language_models/chat_models.py:631, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    628 for i, m in enumerate(messages):
    629     try:
    630         results.append(
--> 631             self._generate_with_cache(
    632                 m,
    633                 stop=stop,
    634                 run_manager=run_managers[i] if run_managers else None,
    635                 **kwargs,
    636             )
    637         )
    638     except BaseException as e:
    639         if run_managers:

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_core/language_models/chat_models.py:853, in BaseChatModel._generate_with_cache(self, messages, stop, run_manager, **kwargs)
    851 else:
    852     if inspect.signature(self._generate).parameters.get(\"run_manager\"):
--> 853         result = self._generate(
    854             messages, stop=stop, run_manager=run_manager, **kwargs
    855         )
    856     else:
    857         result = self._generate(messages, stop=stop, **kwargs)

File ~/.pyenv/versions/3.11.9/envs/cbm/lib/python3.11/site-packages/langchain_ibm/chat_models.py:593, in ChatWatsonx._generate(self, messages, stop, run_manager, stream, **kwargs)
    590 if \"tool_choice\" in kwargs:
    591     del kwargs[\"tool_choice\"]
--> 593 response = self.watsonx_model.generate(
    594     prompt=chat_prompt, **(kwargs | {\"params\": params})
    595 )
    596 return self._create_chat_result(response)

TypeError: ModelInference.generate() got an unexpected keyword argument 'response_format'"
}
langchain-ai / langchain-ibm

ChatWatsonx with Structured output is failing #26
