search

run-llama / llama_index

LlamaIndex is a data framework for your LLM applications
https://docs.llamaindex.ai
MIT License
35.52k stars 5.02k forks source link

[Bug]: cannot use Azure OpenAI with FnRetrieverOpenAIAgent throws LocalProtocolError #10915

Closed pruthviraj90 closed 1 month ago

pruthviraj90 commented 7 months ago

Bug Description

LocalProtocolError: Illegal header value b'Bearer '

The above exception was the direct cause of the following exception:

APIConnectionError Traceback (most recent call last) Cell In[21], line 1 ----> 1 agent.chat("What's 212 multiplied by 122? Make sure to use Tools")

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\llama_index\callbacks\utils.py:41, in trace_method..decorator..wrapper(self, *args, *kwargs) 39 callback_manager = cast(CallbackManager, callback_manager) 40 with callback_manager.as_trace(trace_id): ---> 41 return func(self, args, **kwargs)

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\llama_index\agent\legacy\openai_agent.py:433, in BaseOpenAIAgent.chat(self, message, chat_history, tool_choice) 422 @trace_method("chat") 423 def chat( 424 self, (...) 427 tool_choice: Union[str, dict] = "auto", 428 ) -> AgentChatResponse: 429 with self.callback_manager.event( 430 CBEventType.AGENT_STEP, 431 payload={EventPayload.MESSAGES: [message]}, 432 ) as e: --> 433 chat_response = self._chat( 434 message, chat_history, tool_choice, mode=ChatResponseMode.WAIT 435 ) 436 assert isinstance(chat_response, AgentChatResponse) 437 e.on_end(payload={EventPayload.RESPONSE: chat_response})

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\llama_index\agent\legacy\openai_agent.py:355, in BaseOpenAIAgent._chat(self, message, chat_history, tool_choice, mode) 351 print(f"STARTING TURN {ix}\n---------------\n") 352 llm_chat_kwargs = self._get_llm_chat_kwargs( 353 openai_tools, current_tool_choice 354 ) --> 355 agent_chat_response = self._get_agent_response(mode=mode, **llm_chat_kwargs) 356 if not self._should_continue(self.latest_tool_calls, n_function_calls): 357 logger.debug("Break: should continue False")

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\llama_index\agent\legacy\openai_agent.py:317, in BaseOpenAIAgent._get_agent_response(self, mode, llm_chat_kwargs) 313 def _get_agent_response( 314 self, mode: ChatResponseMode, llm_chat_kwargs: Any 315 ) -> AGENT_CHAT_RESPONSE_TYPE: 316 if mode == ChatResponseMode.WAIT: --> 317 chat_response: ChatResponse = self._llm.chat(**llm_chat_kwargs) 318 return self._process_message(chat_response) 319 elif mode == ChatResponseMode.STREAM:

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\llama_index\llms\base.py:100, in llm_chat_callback..wrap..wrapped_llm_chat(_self, messages, kwargs) 91 with wrapper_logic(_self) as callback_manager: 92 event_id = callback_manager.on_event_start( 93 CBEventType.LLM, 94 payload={ (...) 98 }, 99 ) --> 100 f_return_val = f(_self, messages, kwargs) 102 if isinstance(f_return_val, Generator): 103 # intercept the generator and add a callback to the end 104 def wrapped_gen() -> ChatResponseGen:

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\llama_index\llms\openai.py:237, in OpenAI.chat(self, messages, kwargs) 235 else: 236 chat_fn = completion_to_chat_decorator(self._complete) --> 237 return chat_fn(messages, kwargs)

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\llama_index\llms\openai.py:296, in OpenAI._chat(self, messages, kwargs) 294 client = self._get_client() 295 message_dicts = to_openai_message_dicts(messages) --> 296 response = client.chat.completions.create( 297 messages=message_dicts, 298 stream=False, 299 self._get_model_kwargs(**kwargs), 300 ) 301 openai_message = response.choices[0].message 302 message = from_openai_message(openai_message)

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai_utils_utils.py:275, in required_args..inner..wrapper(*args, *kwargs) 273 msg = f"Missing required argument: {quote(missing[0])}" 274 raise TypeError(msg) --> 275 return func(args, **kwargs)

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai\resources\chat\completions.py:663, in Completions.create(self, messages, model, frequency_penalty, function_call, functions, logit_bias, logprobs, max_tokens, n, presence_penalty, response_format, seed, stop, stream, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout) 611 @required_args(["messages", "model"], ["messages", "model", "stream"]) 612 def create( 613 self, (...) 661 timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, 662 ) -> ChatCompletion | Stream[ChatCompletionChunk]: --> 663 return self._post( 664 "/chat/completions", 665 body=maybe_transform( 666 { 667 "messages": messages, 668 "model": model, 669 "frequency_penalty": frequency_penalty, 670 "function_call": function_call, 671 "functions": functions, 672 "logit_bias": logit_bias, 673 "logprobs": logprobs, 674 "max_tokens": max_tokens, 675 "n": n, 676 "presence_penalty": presence_penalty, 677 "response_format": response_format, 678 "seed": seed, 679 "stop": stop, 680 "stream": stream, 681 "temperature": temperature, 682 "tool_choice": tool_choice, 683 "tools": tools, 684 "top_logprobs": top_logprobs, 685 "top_p": top_p, 686 "user": user, 687 }, 688 completion_create_params.CompletionCreateParams, 689 ), 690 options=make_request_options( 691 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout 692 ), 693 cast_to=ChatCompletion, 694 stream=stream or False, 695 stream_cls=Stream[ChatCompletionChunk], 696 )

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai_base_client.py:1200, in SyncAPIClient.post(self, path, cast_to, body, options, files, stream, stream_cls) 1186 def post( 1187 self, 1188 path: str, (...) 1195 stream_cls: type[_StreamT] | None = None, 1196 ) -> ResponseT | _StreamT: 1197 opts = FinalRequestOptions.construct( 1198 method="post", url=path, json_data=body, files=to_httpx_files(files), **options 1199 ) -> 1200 return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai_base_client.py:889, in SyncAPIClient.request(self, cast_to, options, remaining_retries, stream, stream_cls) 880 def request( 881 self, 882 cast_to: Type[ResponseT], (...) 887 stream_cls: type[_StreamT] | None = None, 888 ) -> ResponseT | _StreamT: --> 889 return self._request( 890 cast_to=cast_to, 891 options=options, 892 stream=stream, 893 stream_cls=stream_cls, 894 remaining_retries=remaining_retries, 895 )

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai_base_client.py:942, in SyncAPIClient._request(self, cast_to, options, remaining_retries, stream, stream_cls) 939 log.debug("Encountered Exception", exc_info=True) 941 if retries > 0: --> 942 return self._retry_request( 943 options, 944 cast_to, 945 retries, 946 stream=stream, 947 stream_cls=stream_cls, 948 response_headers=None, 949 ) 951 log.debug("Raising connection error") 952 raise APIConnectionError(request=request) from err

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai_base_client.py:1013, in SyncAPIClient._retry_request(self, options, cast_to, remaining_retries, response_headers, stream, stream_cls) 1009 # In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a 1010 # different thread if necessary. 1011 time.sleep(timeout) -> 1013 return self._request( 1014 options=options, 1015 cast_to=cast_to, 1016 remaining_retries=remaining, 1017 stream=stream, 1018 stream_cls=stream_cls, 1019 )

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai_base_client.py:942, in SyncAPIClient._request(self, cast_to, options, remaining_retries, stream, stream_cls) 939 log.debug("Encountered Exception", exc_info=True) 941 if retries > 0: --> 942 return self._retry_request( 943 options, 944 cast_to, 945 retries, 946 stream=stream, 947 stream_cls=stream_cls, 948 response_headers=None, 949 ) 951 log.debug("Raising connection error") 952 raise APIConnectionError(request=request) from err

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai_base_client.py:1013, in SyncAPIClient._retry_request(self, options, cast_to, remaining_retries, response_headers, stream, stream_cls) 1009 # In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a 1010 # different thread if necessary. 1011 time.sleep(timeout) -> 1013 return self._request( 1014 options=options, 1015 cast_to=cast_to, 1016 remaining_retries=remaining, 1017 stream=stream, 1018 stream_cls=stream_cls, 1019 )

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai_base_client.py:942, in SyncAPIClient._request(self, cast_to, options, remaining_retries, stream, stream_cls) 939 log.debug("Encountered Exception", exc_info=True) 941 if retries > 0: --> 942 return self._retry_request( 943 options, 944 cast_to, 945 retries, 946 stream=stream, 947 stream_cls=stream_cls, 948 response_headers=None, 949 ) 951 log.debug("Raising connection error") 952 raise APIConnectionError(request=request) from err

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai_base_client.py:1013, in SyncAPIClient._retry_request(self, options, cast_to, remaining_retries, response_headers, stream, stream_cls) 1009 # In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a 1010 # different thread if necessary. 1011 time.sleep(timeout) -> 1013 return self._request( 1014 options=options, 1015 cast_to=cast_to, 1016 remaining_retries=remaining, 1017 stream=stream, 1018 stream_cls=stream_cls, 1019 )

File c:\Users\Admin\AppData\Local\Programs\Python\Python311\Lib\site-packages\openai_base_client.py:952, in SyncAPIClient._request(self, cast_to, options, remaining_retries, stream, stream_cls) 942 return self._retry_request( 943 options, 944 cast_to, (...) 948 response_headers=None, 949 ) 951 log.debug("Raising connection error") --> 952 raise APIConnectionError(request=request) from err 954 log.debug( 955 'HTTP Request: %s %s "%i %s"', request.method, request.url, response.status_code, response.reason_phrase 956 ) 958 try:

APIConnectionError: Connection error.

Version

0.9.33

Steps to Reproduce

from llama_index.tools import FunctionTool from llama_index import ServiceContext, set_global_service_context from llama_index.embeddings import AzureOpenAIEmbedding from llama_index.llms import AzureOpenAI from llama_index import VectorStoreIndex from llama_index.objects import ObjectIndex, SimpleToolNodeMapping from llama_index.agent import FnRetrieverOpenAIAgent

llm = AzureOpenAI( engine=, model=, api_key=, azure_endpoint=, api_version="2023-12-01-preview", )

embed_model = AzureOpenAIEmbedding( model="text-embedding-ada-002", deployment_name=, api_key=, azure_endpoint=, api_version="2023-07-01-preview", )

service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) set_global_service_context(service_context)

def multiply(a: int, b: int) -> int: """Multiply two integers and returns the result integer""" return a * b

def add(a: int, b: int) -> int: """Add two integers and returns the result integer""" return a + b

def useless(a: int, b: int) -> int: """Toy useless function.""" pass

multiply_tool = FunctionTool.from_defaults(fn=multiply, name="multiply") useless_tools = [ FunctionTool.fromdefaults(fn=useless, name=f"useless{str(idx)}") for idx in range(28) ] add_tool = FunctionTool.from_defaults(fn=add, name="add")

all_tools = [multiply_tool] + [add_tool] + useless_tools all_tools_map = {t.metadata.name: t for t in all_tools}

tool_mapping = SimpleToolNodeMapping.from_objects(all_tools) obj_index = ObjectIndex.from_objects( all_tools, tool_mapping, VectorStoreIndex, )

agent = FnRetrieverOpenAIAgent.from_retriever( obj_index.as_retriever(), verbose=True )

agent.chat("What's 212 multiplied by 122? Make sure to use Tools")

Relevant Logs/Tracbacks

No response

logan-markewich commented 7 months ago

@pruthviraj90 this error usually means you did not set an API key

Try passing in the LLM as a kwarg to the fb retriever agent, otherwise it may default to openai

logan-markewich commented 7 months ago 
agent = FnRetrieverOpenAIAgent.from_retriever(
    obj_index.as_retriever(),
    llm=llm, 
    verbose=True
)
WAS-PlaiLabs commented 4 months ago

I get this same error

Traceback (most recent call last):
  File "K:\ComfyUI_win\lib\site-packages\httpx\_transports\default.py", line 66, in map_httpcore_exceptions
    yield
  File "K:\ComfyUI_win\lib\site-packages\httpx\_transports\default.py", line 228, in handle_request
    resp = self._pool.handle_request(req)
  File "K:\ComfyUI_win\lib\site-packages\httpcore\_sync\connection_pool.py", line 216, in handle_request
    raise exc from None
  File "K:\ComfyUI_win\lib\site-packages\httpcore\_sync\connection_pool.py", line 196, in handle_request
    response = connection.handle_request(
  File "K:\ComfyUI_win\lib\site-packages\httpcore\_sync\connection.py", line 101, in handle_request
    return self._connection.handle_request(request)
  File "K:\ComfyUI_win\lib\site-packages\httpcore\_sync\http11.py", line 143, in handle_request
    raise exc
  File "K:\ComfyUI_win\lib\site-packages\httpcore\_sync\http11.py", line 93, in handle_request
    self._send_request_headers(**kwargs)
  File "K:\ComfyUI_win\lib\site-packages\httpcore\_sync\http11.py", line 151, in _send_request_headers
    with map_exceptions({h11.LocalProtocolError: LocalProtocolError}):
  File "C:\Users\jorda\miniconda3\lib\contextlib.py", line 153, in __exit__
    self.gen.throw(typ, value, traceback)
  File "K:\ComfyUI_win\lib\site-packages\httpcore\_exceptions.py", line 14, in map_exceptions
    raise to_exc(exc) from exc
httpcore.LocalProtocolError: Illegal header value b'Bearer '

Except, it's on the .complete() method of the LLM model itself, which is a OpenAIMultiModel with image_documnets being passed a list of ImageDocument objects.

Is this related?