from openai import OpenAI
import sys
port=sys.argv[1] if len(sys.argv) > 1 else 8000
model=sys.argv[2] if len(sys.argv) > 2 else "gpt-3.5-turbo"
api_base = f'localhost:{port}/v1'
client = OpenAI(base_url=api_base, api_key="xxx")
while True:
import time
start = time.time()
# 调用 chat.completions.create 方法并启用流式接口
response = client.chat.completions.create(
model=model,
messages=[
# {"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "讲一个微小说,500字以内。"}
],
stream=True # 启用流式接口
)
# 逐步接收数据
idx = 0
for chunk in response:
end = time.time()
if idx == 0:
print(f"time:{end-start}")
print(chunk.choices[0].delta.content)
start = time.time()
idx += 1
time.sleep(.1)
step
1. start server
2. start two client
* python client.py 8000 lora1
* python client.py 8000 lora2
3. stop one client
bug log
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/uvicorn/protocols/http/httptools_impl.py", line 411, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 69, in call
return await self.app(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/fastapi/applications.py", line 1054, in call
await super().call(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/applications.py", line 123, in call
await self.middleware_stack(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/errors.py", line 186, in call
raise exc
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/errors.py", line 164, in call
await self.app(scope, receive, _send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/cors.py", line 85, in call
await self.app(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 65, in call
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 756, in call
await self.middleware_stack(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 776, in app
await route.handle(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 297, in handle
await self.app(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 77, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 75, in app
await response(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 258, in call
async with anyio.create_task_group() as task_group:
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 597, in aexit
raise exceptions[0]
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 261, in wrap
await func()
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 250, in stream_response
async for chunk in self.body_iterator:
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_chat.py", line 228, in chat_completion_stream_generator
async for res in result_generator:
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 772, in generate
async for output in self._process_request(
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 888, in _process_request
raise e
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 884, in _process_request
async for request_output in stream:
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 93, in anext
raise result
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 46, in _log_task_completion
return_value = task.result()
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 630, in run_engine_loop
async with asyncio_timeout(ENGINE_ITERATION_TIMEOUT_S):
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_timeout.py", line 95, in aexit
self._do_exit(exc_type)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_timeout.py", line 178, in _do_exit
raise asyncio.TimeoutError
asyncio.exceptions.TimeoutError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/uvicorn/protocols/http/httptools_impl.py", line 411, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 69, in call
return await self.app(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/fastapi/applications.py", line 1054, in call
await super().call(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/applications.py", line 123, in call
await self.middleware_stack(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/errors.py", line 186, in call
raise exc
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/errors.py", line 164, in call
await self.app(scope, receive, _send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/cors.py", line 85, in call
await self.app(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 65, in call
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 756, in call
await self.middleware_stack(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 776, in app
await route.handle(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 297, in handle
await self.app(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 77, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 75, in app
await response(scope, receive, send)
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 258, in call
async with anyio.create_task_group() as task_group:
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 597, in aexit
raise exceptions[0]
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 261, in wrap
await func()
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 250, in stream_response
async for chunk in self.body_iterator:
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_chat.py", line 228, in chat_completion_stream_generator
async for res in result_generator:
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 772, in generate
async for output in self._process_request(
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 873, in _process_request
stream = await self.add_request(
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 676, in add_request
self.start_background_loop()
File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 516, in start_background_loop
raise AsyncEngineDeadError(
vllm.engine.async_llm_engine.AsyncEngineDeadError: Background loop has errored already.
Your current environment
🐛 Describe the bug
server
vllm serve qwen/Qwen2-7B-Instruct --port 8000 --served-model-name gpt-3.5-turbo --disable-log-stats --tensor-parallel-size 4 --gpu-memory-utilization 0.25 --enable-lora --lora-modules lora1=saves/qwen2_lora1/lora/sft lora2=saves/qwen2_lora2/lora/sft
client(client.py)
step
bug log
ERROR: Exception in ASGI application Traceback (most recent call last): File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/uvicorn/protocols/http/httptools_impl.py", line 411, in run_asgi result = await app( # type: ignore[func-returns-value] File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 69, in call return await self.app(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/fastapi/applications.py", line 1054, in call await super().call(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/applications.py", line 123, in call await self.middleware_stack(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/errors.py", line 186, in call raise exc File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/errors.py", line 164, in call await self.app(scope, receive, _send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/cors.py", line 85, in call await self.app(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 65, in call await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 756, in call await self.middleware_stack(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 776, in app await route.handle(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 297, in handle await self.app(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 77, in app await wrap_app_handling_exceptions(app, request)(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 75, in app await response(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 258, in call async with anyio.create_task_group() as task_group: File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 597, in aexit raise exceptions[0] File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 261, in wrap await func() File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 250, in stream_response async for chunk in self.body_iterator: File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_chat.py", line 228, in chat_completion_stream_generator async for res in result_generator: File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 772, in generate async for output in self._process_request( File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 888, in _process_request raise e File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 884, in _process_request async for request_output in stream: File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 93, in anext raise result File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 46, in _log_task_completion return_value = task.result() File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 630, in run_engine_loop async with asyncio_timeout(ENGINE_ITERATION_TIMEOUT_S): File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_timeout.py", line 95, in aexit self._do_exit(exc_type) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_timeout.py", line 178, in _do_exit raise asyncio.TimeoutError asyncio.exceptions.TimeoutError
The above exception was the direct cause of the following exception:
Traceback (most recent call last): File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/uvicorn/protocols/http/httptools_impl.py", line 411, in run_asgi result = await app( # type: ignore[func-returns-value] File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 69, in call return await self.app(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/fastapi/applications.py", line 1054, in call await super().call(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/applications.py", line 123, in call await self.middleware_stack(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/errors.py", line 186, in call raise exc File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/errors.py", line 164, in call await self.app(scope, receive, _send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/cors.py", line 85, in call await self.app(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 65, in call await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 756, in call await self.middleware_stack(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 776, in app await route.handle(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 297, in handle await self.app(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 77, in app await wrap_app_handling_exceptions(app, request)(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/routing.py", line 75, in app await response(scope, receive, send) File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 258, in call async with anyio.create_task_group() as task_group: File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 597, in aexit raise exceptions[0] File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 261, in wrap await func() File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/starlette/responses.py", line 250, in stream_response async for chunk in self.body_iterator: File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_chat.py", line 228, in chat_completion_stream_generator async for res in result_generator: File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 772, in generate async for output in self._process_request( File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 873, in _process_request stream = await self.add_request( File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 676, in add_request self.start_background_loop() File "/home/jovyan/conda-env/envs/xqh_vllm/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 516, in start_background_loop raise AsyncEngineDeadError( vllm.engine.async_llm_engine.AsyncEngineDeadError: Background loop has errored already.