[Bug]: vllm 启动，openai的swarm 函数调用不正常

18600709862 commented 4 weeks ago

Model Series

Qwen2.5

What are the models used?

Qwen2.5-72B-Instruct-AWQ

What is the scenario where the problem happened?

vllm启动

Is this a known issue?

[X] I have followed the GitHub README.
[X] I have checked the Qwen documentation and cannot find an answer there.
[X] I have checked the documentation of the related framework and cannot find useful information.
[X] I have searched the issues and there is not a similar one.

Information about environment

OS: Ubuntu 22.04 Python: Python 3.11 GPUs: 4 x NVIDIA A20 NVIDIA driver: 535 (from nvidia-smi) CUDA compiler: 12.1 (from nvcc -V) PyTorch: 2.4.1+cu121 (from python -c "import troch; print(torch.version)")

Log output

错误信息如下：
openai.BadRequestError: Error code: 400 - {'object': 'error', 'message': '[{\'type\': \'string_type\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'content\', \'str\'), \'msg\': \'Input should be a valid string\', \'input\': None}, {\'type\': \'iterable_type\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'content\', \'generator[typed-dict]\'), \'msg\': \'Input should be iterable\', \'input\': None}, {\'type\': \'literal_error\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'role\'), \'msg\': "Input should be \'system\'", \'input\': \'assistant\', \'ctx\': {\'expected\': "\'system\'"}}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'refusal\'), \'msg\': \'Extra inputs are not permitted\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'function_call\'), \'msg\': \'Extra inputs are not permitted\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'tool_calls\'), \'msg\': \'Extra inputs are not permitted\', \'input\': [{\'id\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\', \'function\': {\'arguments\': \'{"location": "NYC"}\', \'name\': \'get_weather\'}, \'type\': \'function\'}]}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'sender\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'Agent\'}, {\'type\': \'string_type\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'content\', \'str\'), \'msg\': \'Input should be a valid string\', \'input\': None}, {\'type\': \'iterable_type\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'content\', \'generator[union[typed-dict,typed-dict]]\'), \'msg\': \'Input should be iterable\', \'input\': None}, {\'type\': \'literal_error\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'role\'), \'msg\': "Input should be \'user\'", \'input\': \'assistant\', \'ctx\': {\'expected\': "\'user\'"}}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'refusal\'), \'msg\': \'Extra inputs are not permitted\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'function_call\'), \'msg\': \'Extra inputs are not permitted\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'tool_calls\'), \'msg\': \'Extra inputs are not permitted\', \'input\': [{\'id\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\', \'function\': {\'arguments\': \'{"location": "NYC"}\', \'name\': \'get_weather\'}, \'type\': \'function\'}]}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'sender\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'Agent\'}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'sender\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'Agent\'}, {\'type\': \'string_type\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'content\', \'str\'), \'msg\': \'Input should be a valid string\', \'input\': None}, {\'type\': \'iterable_type\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'content\', \'generator[typed-dict]\'), \'msg\': \'Input should be iterable\', \'input\': None}, {\'type\': \'literal_error\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'role\'), \'msg\': "Input should be \'tool\'", \'input\': \'assistant\', \'ctx\': {\'expected\': "\'tool\'"}}, {\'type\': \'missing\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'tool_call_id\'), \'msg\': \'Field required\', \'input\': {\'content\': None, \'refusal\': None, \'role\': \'assistant\', \'function_call\': None, \'tool_calls\': [{\'id\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\', \'function\': {\'arguments\': \'{"location": "NYC"}\', \'name\': \'get_weather\'}, \'type\': \'function\'}], \'sender\': \'Agent\'}}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'refusal\'), \'msg\': \'Extra inputs are not permitted\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'function_call\'), \'msg\': \'Extra inputs are not permitted\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'tool_calls\'), \'msg\': \'Extra inputs are not permitted\', \'input\': [{\'id\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\', \'function\': {\'arguments\': \'{"location": "NYC"}\', \'name\': \'get_weather\'}, \'type\': \'function\'}]}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'sender\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'Agent\'}, {\'type\': \'missing\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'name\'), \'msg\': \'Field required\', \'input\': {\'content\': None, \'refusal\': None, \'role\': \'assistant\', \'function_call\': None, \'tool_calls\': [{\'id\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\', \'function\': {\'arguments\': \'{"location": "NYC"}\', \'name\': \'get_weather\'}, \'type\': \'function\'}], \'sender\': \'Agent\'}}, {\'type\': \'literal_error\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'role\'), \'msg\': "Input should be \'function\'", \'input\': \'assistant\', \'ctx\': {\'expected\': "\'function\'"}}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'refusal\'), \'msg\': \'Extra inputs are not permitted\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'function_call\'), \'msg\': \'Extra inputs are not permitted\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'tool_calls\'), \'msg\': \'Extra inputs are not permitted\', \'input\': [{\'id\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\', \'function\': {\'arguments\': \'{"location": "NYC"}\', \'name\': \'get_weather\'}, \'type\': \'function\'}]}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'sender\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'Agent\'}, {\'type\': \'string_type\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'content\', \'str\'), \'msg\': \'Input should be a valid string\', \'input\': None}, {\'type\': \'list_type\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'content\', \'list[union[typed-dict,typed-dict,typed-dict,typed-dict,typed-dict]]\'), \'msg\': \'Input should be a valid list\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'refusal\'), \'msg\': \'Extra inputs are not permitted\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'function_call\'), \'msg\': \'Extra inputs are not permitted\', \'input\': None}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 2, \'typed-dict\', \'sender\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'Agent\'}, {\'type\': \'literal_error\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'role\'), \'msg\': "Input should be \'system\'", \'input\': \'tool\', \'ctx\': {\'expected\': "\'system\'"}}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'tool_call_id\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\'}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'tool_name\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'get_weather\'}, {\'type\': \'literal_error\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'role\'), \'msg\': "Input should be \'user\'", \'input\': \'tool\', \'ctx\': {\'expected\': "\'user\'"}}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'tool_call_id\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\'}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'tool_name\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'get_weather\'}, {\'type\': \'literal_error\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'role\'), \'msg\': "Input should be \'assistant\'", \'input\': \'tool\', \'ctx\': {\'expected\': "\'assistant\'"}}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'tool_call_id\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\'}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'tool_name\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'get_weather\'}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'tool_name\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'get_weather\'}, {\'type\': \'missing\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'name\'), \'msg\': \'Field required\', \'input\': {\'role\': \'tool\', \'tool_call_id\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\', \'tool_name\': \'get_weather\', \'content\': "{\'temp\':67, \'unit\':\'F\'}"}}, {\'type\': \'literal_error\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'role\'), \'msg\': "Input should be \'function\'", \'input\': \'tool\', \'ctx\': {\'expected\': "\'function\'"}}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'tool_call_id\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'chatcmpl-tool-db309d8c2d1d4dbb993017c0330862a8\'}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'tool_name\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'get_weather\'}, {\'type\': \'extra_forbidden\', \'loc\': (\'body\', \'messages\', 3, \'typed-dict\', \'tool_name\'), \'msg\': \'Extra inputs are not permitted\', \'input\': \'get_weather\'}]', 'type': 'BadRequestError', 'param': None, 'code': 400}

Description

Steps to reproduce

Qwen2.5-72B-Instruct-AWQ

部署验证openai的swarm 地址https://github.com/openai/swarm

swarm/examples/basic
1、vim function_calling.py

from swarm import Swarm, Agent
from openai import OpenAI
#client = OpenAI(api_key="empty",base_url="http://192.168.10.3:11434/v1")
client = OpenAI(api_key="empty",base_url="http://192.168.10.3:8000/v1")
client = Swarm(client)

def get_weather(location) -> str:
    return "{'temp':67, 'unit':'F'}"

model = "gpt-4-turbo"
#model = "qwen2.5:72b"
agent = Agent(
    name="Agent",
    model=model,
    instructions="You are a helpful agent.",
    tool_choice="auto",
    parallel_tool_calls=False,
    functions=[get_weather],
)

messages = [{"role": "user", "content": "What's the weather in NYC?"}]

#response = client.run(agent=agent, model_override="qwen2.5:72b",messages=messages)

response = client.run(
    agent=agent,
    messages=messages)
print(response.messages[-1]["content"])

python function_calling.py

Expected results

使用ollama运行 qwen2.5:72b 可以正确得到调用结果

The current temperature in NYC is 67°F.

然而使用vllm

 vllm serve Qwen/Qwen2.5-72B-Instruct-AWQ --tensor-parallel-size=8 --quantization=awq  --enable-auto-tool-choice --tool-call-parser hermes

不能得到调用结果

日志显示

INFO:     192.168.10.3:53526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
INFO:     192.168.10.3:53526 - "POST /v1/chat/completions HTTP/1.1" 400 Bad Request

这里是否是vllm启动参数设置问题，还是没有正确启动。

Attempts to fix

 vllm serve Qwen/Qwen2.5-72B-Instruct-AWQ --tensor-parallel-size=8 --quantization=awq  --enable-auto-tool-choice --tool-call-parser hermes   --chat-template template.jinja

没有效果

Anything else helpful for investigation

proxy2008 commented 2 weeks ago

Running this example encountered the same problem.

guoping1127 commented 1 week ago

我也是这样报这个错误有人解决吗？

six-six-six-six commented 14 hours ago

我发现了一个简陋的解决方案。

首先说明一下我认为的问题所在：

通过debug观察swarm的执行记录并对比qwen-agent提供的vllm执行记录可以发现swarm在执行过程中对message参数增加了一些信息，因此我们需要做的是消除这些冗余信息就行。

解决方法

version: 0.1.1

修改swarm/core.py文件

1.注释109、129行的
```
"tool_name": name,
```

2.修改ru()函数，如下：

def run(
    self,
    agent: Agent,
    messages: List,
    context_variables: dict = {},
    model_override: str = None,
    stream: bool = False,
    debug: bool = False,
    max_turns: int = float("inf"),
    execute_tools: bool = True,
) -> Response:
if stream:
    return self.run_and_stream(
        agent=agent,
        messages=messages,
        context_variables=context_variables,
        model_override=model_override,
        debug=debug,
        max_turns=max_turns,
        execute_tools=execute_tools,
    )
active_agent = agent
context_variables = copy.deepcopy(context_variables)
history = copy.deepcopy(messages)
history_for_show = copy.deepcopy(messages)
init_len = len(messages)

while len(history) - init_len < max_turns and active_agent:

    # get completion with current history, agent
    completion = self.get_chat_completion(
        agent=active_agent,
        history=history,
        context_variables=context_variables,
        model_override=model_override,
        stream=stream,
        debug=debug,
    )
    message = completion.choices[0].message
    debug_print(debug, "Received completion:", message)
    # message.sender = active_agent.name
    history.append(
        json.loads(message.model_dump_json())
    )  # to avoid OpenAI types (?)

    message_for_show = copy.deepcopy(message)
    message_for_show.sender = active_agent.name
    history_for_show.append(
        json.loads(message_for_show.model_dump_json())
    )

    if not message.tool_calls or not execute_tools:
        debug_print(debug, "Ending turn.")
        break

    # handle function calls, updating context_variables, and switching agents
    partial_response = self.handle_tool_calls(
        message.tool_calls, active_agent.functions, context_variables, debug
    )
    history.extend(partial_response.messages)
    history_for_show.extend(partial_response.messages)
    context_variables.update(partial_response.context_variables)
    if partial_response.agent:
        active_agent = partial_response.agent

return Response(
    messages=history_for_show[init_len:],
    agent=active_agent,
    context_variables=context_variables,
)

修改后的执行效果（swarm/examples/weather_agent）：

QwenLM / Qwen2.5