Internal server error after the second method call to ChatSession#send_message

[nicoloboschi]

Thanks for stopping by to let us know something could be better!

Environment details

• OS type and version: MacOS / linux
• Python version: 3.11
• pip version: doesn't matter
• google-cloud-aiplatform version: 1.39.0

Steps to reproduce

Run the below code. This happens systematically at the second execution of the same method. If you send text-only (for example using gemini-pro) it works as expected

Code example

from vertexai.preview import generative_models

def test_send_message_from_list_of_parts():
        model = generative_models.GenerativeModel("gemini-pro-vision")
        chat = model.start_chat()
        image_part = generative_models.Part.from_uri(
            uri="gs://download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg",
            mime_type="image/jpeg",
        )
        text_part = generative_models.Part.from_text(
            "What is shown in this image?"
        )
        response1 = chat.send_message([text_part, image_part])
        assert response1.text
        assert len(chat.history) == 2

        response2 = chat.send_message([text_part, image_part])

Stack trace

tests/system/vertexai/test_generative_models.py:163 (TestGenerativeModels.test_send_message_from_list_of_parts)
args = (model: "projects/datastax-gcp-pulsar/locations/us-central1/publishers/google/models/gemini-pro-vision"
contents {
  r...ge/jpeg"
      file_uri: "gs://download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg"
    }
  }
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'model=projects/datastax-gcp-pulsar/locations/us-central1/publishers/google/mo...0 gapic/1.39.0+top_google_constructor_method+vertexai.generative_models._generative_models.ChatSession.send_message')]}
result = <_MultiThreadedRendezvous of RPC that terminated with:
    status = StatusCode.INTERNAL
    details = "Internal error encoun...70:443 {created_time:"2024-01-24T18:25:32.257084+01:00", grpc_status:13, grpc_message:"Internal error encountered."}"
>
prefetch_first = True

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
            result = callable_(*args, **kwargs)
            # Auto-fetching the first result causes PubSub client's streaming pull
            # to hang when re-opening the stream, thus we need examine the hacky
            # hidden flag to see if pre-fetching is disabled.
            # https://github.com/googleapis/python-pubsub/issues/93#issuecomment-630762257
            prefetch_first = getattr(callable_, "_prefetch_first_result_", True)
>           return _StreamingResponseIterator(
                result, prefetch_first_result=prefetch_first
            )

../../../venv/lib/python3.11/site-packages/google/api_core/grpc_helpers.py:173: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../../venv/lib/python3.11/site-packages/google/api_core/grpc_helpers.py:95: in __init__
    self._stored_first_result = next(self._wrapped)
../../../venv/lib/python3.11/site-packages/grpc/_channel.py:540: in __next__
    return self._next()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_MultiThreadedRendezvous of RPC that terminated with:
    status = StatusCode.INTERNAL
    details = "Internal error encoun...70:443 {created_time:"2024-01-24T18:25:32.257084+01:00", grpc_status:13, grpc_message:"Internal error encountered."}"
>

    def _next(self) -> Any:
        with self._state.condition:
            if self._state.code is None:
                event_handler = _event_handler(
                    self._state, self._response_deserializer
                )
                self._state.due.add(cygrpc.OperationType.receive_message)
                operating = self._call.operate(
                    (cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS),),
                    event_handler,
                )
                if not operating:
                    self._state.due.remove(cygrpc.OperationType.receive_message)
            elif self._state.code is grpc.StatusCode.OK:
                raise StopIteration()
            else:
                raise self

            def _response_ready():
                return self._state.response is not None or (
                    cygrpc.OperationType.receive_message not in self._state.due
                    and self._state.code is not None
                )

            _common.wait(self._state.condition.wait, _response_ready)
            if self._state.response is not None:
                response = self._state.response
                self._state.response = None
                return response
            elif cygrpc.OperationType.receive_message not in self._state.due:
                if self._state.code is grpc.StatusCode.OK:
                    raise StopIteration()
                elif self._state.code is not None:
>                   raise self
E                   grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
E                       status = StatusCode.INTERNAL
E                       details = "Internal error encountered."
E                       debug_error_string = "UNKNOWN:Error received from peer ipv4:142.250.180.170:443 {created_time:"2024-01-24T18:25:32.257084+01:00", grpc_status:13, grpc_message:"Internal error encountered."}"
E                   >

../../../venv/lib/python3.11/site-packages/grpc/_channel.py:966: _MultiThreadedRendezvous

The above exception was the direct cause of the following exception:

self = <test_generative_models.TestGenerativeModels object at 0x137010290>

    def test_send_message_from_list_of_parts(self):

        model = generative_models.GenerativeModel("gemini-pro-vision")
        chat = model.start_chat()
        image_part = generative_models.Part.from_uri(
            uri="gs://download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg",
            mime_type="image/jpeg",
        )
        text_part = generative_models.Part.from_text(
            "What is shown in this image?"
        )
        response1 = chat.send_message([text_part, image_part])
        assert response1.text
        assert len(chat.history) == 2

>       response1 = chat.send_message([text_part, image_part])

test_generative_models.py:179: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../../vertexai/generative_models/_generative_models.py:710: in send_message
    return self._send_message(
../../../vertexai/generative_models/_generative_models.py:797: in _send_message
    response = self._model._generate_content(
../../../vertexai/generative_models/_generative_models.py:438: in _generate_content
    stream = self._prediction_client.stream_generate_content(request=request)
../../../google/cloud/aiplatform_v1beta1/services/prediction_service/client.py:1634: in stream_generate_content
    response = rpc(
../../../venv/lib/python3.11/site-packages/google/api_core/gapic_v1/method.py:131: in __call__
    return wrapped_func(*args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = (model: "projects/datastax-gcp-pulsar/locations/us-central1/publishers/google/models/gemini-pro-vision"
contents {
  r...ge/jpeg"
      file_uri: "gs://download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg"
    }
  }
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'model=projects/datastax-gcp-pulsar/locations/us-central1/publishers/google/mo...0 gapic/1.39.0+top_google_constructor_method+vertexai.generative_models._generative_models.ChatSession.send_message')]}
result = <_MultiThreadedRendezvous of RPC that terminated with:
    status = StatusCode.INTERNAL
    details = "Internal error encoun...70:443 {created_time:"2024-01-24T18:25:32.257084+01:00", grpc_status:13, grpc_message:"Internal error encountered."}"
>
prefetch_first = True

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
            result = callable_(*args, **kwargs)
            # Auto-fetching the first result causes PubSub client's streaming pull
            # to hang when re-opening the stream, thus we need examine the hacky
            # hidden flag to see if pre-fetching is disabled.
            # https://github.com/googleapis/python-pubsub/issues/93#issuecomment-630762257
            prefetch_first = getattr(callable_, "_prefetch_first_result_", True)
            return _StreamingResponseIterator(
                result, prefetch_first_result=prefetch_first
            )
        except grpc.RpcError as exc:
>           raise exceptions.from_grpc_error(exc) from exc
E           google.api_core.exceptions.InternalServerError: 500 Internal error encountered.

../../../venv/lib/python3.11/site-packages/google/api_core/grpc_helpers.py:177: InternalServerError

[ZhenyiQ]

Hi @nicoloboschi ,

The gemini-pro-vision doesn't yet support multimodal multiturn.

The error is indeed not informative at all. We'll bring this up to the API team and hopefully this can be resolved soon. We don't plan to change the error message on the SDK level since 1) what the model accepts may change in the future and 2) internal error can mean a lot of things. Apologize for the confusion!