gradio-app / gradio

Build and share delightful machine learning apps, all in Python. 🌟 Star to support our work!
http://www.gradio.app
Apache License 2.0
32.27k stars 2.41k forks source link

ValueError when using gradio components in Chatbot messages and streaming response #9366

Open lippings opened 4 days ago

lippings commented 4 days ago

Describe the bug

I have a chatbot implemented using gradio.Chatbot, where I am streaming the chatbot's response. I also want the chatbot to be able to include gradio components in it's responses, as well as text. I can get the component to show up fine, but the next yield from the generator function after the component is shown causes a ValueError.

Have you searched existing issues? πŸ”Ž

Reproduction

import gradio as gr

def add_msg(history, human_input):
    msg = gr.ChatMessage(
        role='user',
        content=human_input
    )
    history.append(msg)

    return history, ''

def fn(history):
    history.append(gr.ChatMessage(
        role='assistant',
        content='Here\'s'
    ))
    yield history

    history.append(gr.ChatMessage(
        role='assistant',
        content='an image:'
    ))
    yield history

    history.append(gr.ChatMessage(
        role='assistant',
        content=gr.Image(
            'https://www.startpage.com/av/proxy-image?piurl=https%3A%2F%2Ftse3.mm.bing.net%2Fth%3Fid%3DOIP.2GCSsS1HKwRa6qWotlHXHQHaEK%26pid%3DApi&sp=1726513283T319971ef4351fe251279b3b562767806a0ddbbde1d1c96ced3168dd7e811bbad'
        )
    ))

    yield history

    history.append(gr.ChatMessage(
        role='assistant',
        content='Wow, what an image!'
    ))
    yield history

with gr.Blocks(fill_height=True) as demo:
    chatbot = gr.Chatbot(type='messages', scale=1)
    human_input = gr.Textbox()

    human_input.submit(
        add_msg, [chatbot, human_input], [chatbot, human_input]
    ).then(
        fn, chatbot, chatbot
    )

demo.launch()

The bot should respond with text, an image, and more text after that.

Screenshot

gradio_chatbot_component_error

There is an error before the last piece of text, immediately after the image.

Logs

Running on local URL:  http://127.0.0.1:7860

To create a public link, set `share=True` in `launch()`.
Traceback (most recent call last):
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/queueing.py", line 575, in process_events
    response = await route_utils.call_process_api(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/route_utils.py", line 322, in call_process_api
    output = await app.get_blocks().process_api(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/blocks.py", line 1945, in process_api
    data = await self.postprocess_data(block_fn, result["prediction"], state)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/blocks.py", line 1768, in postprocess_data
    prediction_value = block.postprocess(prediction_value)
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/components/chatbot.py", line 496, in postprocess
    processed_messages = [
                         ^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/components/chatbot.py", line 499, in <listcomp>
    for msg in self._postprocess_message_messages(cast(MessageDict, message))
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/components/chatbot.py", line 441, in _postprocess_message_messages
    message.content = self._postprocess_content(message.content)  # type: ignore
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/components/chatbot.py", line 421, in _postprocess_content
    raise ValueError(f"Invalid message for Chatbot component: {chat_message}")
ValueError: Invalid message for Chatbot component: component='image' value={'path': '/tmp/gradio/2e96eca335c0769fe93a7287d9342bca0c0708742adeaa66786bb928838d41a6/proxy-imagepiurlhttps3A2F2Ftse3.mm.bing.net2Fth3Fid3DOIP.2GCSsS1HKwRa6qWotlHXHQHaEK26pid3DApisp1726513283T319971ef4351fe251279b3b562767806a0ddbbde1d1c96ced3168dd7e811bbad', 'url': '/file=/tmp/gradio/2e96eca335c0769fe93a7287d9342bca0c0708742adeaa66786bb928838d41a6/proxy-imagepiurlhttps3A2F2Ftse3.mm.bing.net2Fth3Fid3DOIP.2GCSsS1HKwRa6qWotlHXHQHaEK26pid3DApisp1726513283T319971ef4351fe251279b3b562767806a0ddbbde1d1c96ced3168dd7e811bbad', 'size': None, 'orig_name': None, 'mime_type': None, 'is_stream': False, 'meta': {'_type': 'gradio.FileData'}} constructor_args={} props={'streamable': False, 'value': {'path': '/tmp/gradio/2e96eca335c0769fe93a7287d9342bca0c0708742adeaa66786bb928838d41a6/proxy-imagepiurlhttps3A2F2Ftse3.mm.bing.net2Fth3Fid3DOIP.2GCSsS1HKwRa6qWotlHXHQHaEK26pid3DApisp1726513283T319971ef4351fe251279b3b562767806a0ddbbde1d1c96ced3168dd7e811bbad', 'url': '/file=/tmp/gradio/2e96eca335c0769fe93a7287d9342bca0c0708742adeaa66786bb928838d41a6/proxy-imagepiurlhttps3A2F2Ftse3.mm.bing.net2Fth3Fid3DOIP.2GCSsS1HKwRa6qWotlHXHQHaEK26pid3DApisp1726513283T319971ef4351fe251279b3b562767806a0ddbbde1d1c96ced3168dd7e811bbad', 'size': None, 'orig_name': None, 'mime_type': None, 'is_stream': False, 'meta': {'_type': 'gradio.FileData'}}, 'format': 'webp', 'height': None, 'width': None, 'image_mode': 'RGB', 'sources': ['upload', 'webcam', 'clipboard'], 'type': 'numpy', 'label': None, 'show_label': True, 'show_download_button': True, 'container': True, 'scale': None, 'min_width': 160, 'interactive': None, 'visible': True, 'streaming': False, 'elem_id': None, 'elem_classes': [], 'key': None, 'mirror_webcam': True, 'show_share_button': False, 'placeholder': None, 'show_fullscreen_button': True, 'proxy_url': None, 'name': 'image', '_selectable': False}

System Info

Gradio Environment Information:
------------------------------
Operating System: Linux
gradio version: 4.44.0
gradio_client version: 1.3.0

------------------------------------------------
gradio dependencies in your environment:

aiofiles: 23.2.1
anyio: 4.4.0
fastapi: 0.114.2
ffmpy: 0.4.0
gradio-client==1.3.0 is not installed.
httpx: 0.27.2
huggingface-hub: 0.24.7
importlib-resources: 6.4.5
jinja2: 3.1.4
markupsafe: 2.1.5
matplotlib: 3.9.2
numpy: 2.1.1
orjson: 3.10.7
packaging: 24.1
pandas: 2.2.2
pillow: 10.4.0
pydantic: 2.9.1
pydub: 0.25.1
python-multipart: 0.0.9
pyyaml: 6.0.2
ruff: 0.6.5
semantic-version: 2.10.0
tomlkit==0.12.0 is not installed.
typer: 0.12.5
typing-extensions: 4.12.2
urllib3: 2.2.3
uvicorn: 0.30.6
authlib; extra == 'oauth' is not installed.
itsdangerous; extra == 'oauth' is not installed.

gradio_client dependencies in your environment:

fsspec: 2024.9.0
httpx: 0.27.2
huggingface-hub: 0.24.7
packaging: 24.1
typing-extensions: 4.12.2
websockets: 12.0

Severity

I can work around it

lippings commented 4 days ago

I can work around this by adding an if-clause for ComponentMessage in Chatbot._postprocess_content:

def _postprocess_content(
        self,
        chat_message: str
        | tuple
        | list
        | FileDataDict
        | FileData
        | GradioComponent
        | None,
    ) -> str | FileMessage | ComponentMessage | None:
        if chat_message is None:
            return None
        elif isinstance(chat_message, FileMessage):
            return chat_message
        elif isinstance(chat_message, FileData):
            return FileMessage(file=chat_message)
        elif isinstance(chat_message, ComponentMessage):  # <------------ This is the added clause
            return chat_message
        elif isinstance(chat_message, GradioComponent):
            component = import_component_and_data(type(chat_message).__name__)
            if component:
                component = chat_message.__class__(**chat_message.constructor_args)
                chat_message.constructor_args.pop("value", None)
                config = component.get_config()
                return ComponentMessage(
                    component=type(chat_message).__name__.lower(),
                    value=config.get("value", None),
                    constructor_args=chat_message.constructor_args,
                    props=config,
                )
        elif isinstance(chat_message, dict) and "path" in chat_message:
            filepath = chat_message["path"]
            return self._create_file_message(chat_message, filepath)
        elif isinstance(chat_message, (tuple, list)):
            filepath = str(chat_message[0])
            return self._create_file_message(chat_message, filepath)
        elif isinstance(chat_message, str):
            chat_message = inspect.cleandoc(chat_message)
            return chat_message
        else:
            raise ValueError(f"Invalid message for Chatbot component: {chat_message}")

But I'm not sure this is the "proper" way to fix this, since I'm not too familiar with Gradio's internals.