search

googleapis / google-cloud-java

Google Cloud Client Library for Java
https://cloud.google.com/java/docs/reference
Apache License 2.0
1.86k stars 1.06k forks source link

VertexAI Nightly Failure: RESOURCE_EXHAUSTED #10983

Open burkedavison opened 1 week ago

burkedavison commented 1 week ago

https://fusion2.corp.google.com/invocations/c0ae4b14-5bad-4ab7-a904-78f962a9e0be/targets/cloud-devrel%2Fclient-libraries%2Fjava%2Fgoogle-cloud-java%2Fnightly%2Fgraalvm-sub-jobs%2Fnative-b%2Fgraalvm-native-1;config=default/log and https://fusion2.corp.google.com/invocations/4ce01b64-80fc-43f6-8189-355d1b5a7778/targets/cloud-devrel%2Fclient-libraries%2Fjava%2Fgoogle-cloud-java%2Fnightly%2Fgraalvm-sub-jobs%2Fnative-a%2Fgraalvm-native-1;config=default/log

08:16:48:260 [ERROR] com.google.cloud.vertexai.it.ITChatSessionIntegrationTest.sendMessageWithNewConfigs_historyContainsFullConversation -- Time elapsed: 11.32 s <<< ERROR!
com.google.api.gax.rpc.ResourceExhaustedException: io.grpc.StatusRuntimeException: RESOURCE_EXHAUSTED: Unable to submit request because the service is temporarily out of capacity. Try again later.
    at com.google.api.gax.rpc.ApiExceptionFactory.createException(ApiExceptionFactory.java:100)
    at com.google.api.gax.rpc.ApiExceptionFactory.createException(ApiExceptionFactory.java:41)
    at com.google.api.gax.grpc.GrpcApiExceptionFactory.create(GrpcApiExceptionFactory.java:86)
    at com.google.api.gax.grpc.GrpcApiExceptionFactory.create(GrpcApiExceptionFactory.java:66)
    at com.google.api.gax.grpc.GrpcExceptionCallable$ExceptionTransformingFuture.onFailure(GrpcExceptionCallable.java:97)
    at com.google.api.core.ApiFutures$1.onFailure(ApiFutures.java:84)
    at com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1130)
    at com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
    at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1298)
    at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1059)
    at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:809)
    at io.grpc.stub.ClientCalls$GrpcFuture.setException(ClientCalls.java:568)
    at io.grpc.stub.ClientCalls$UnaryStreamToFuture.onClose(ClientCalls.java:538)
    at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
    at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
    at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
    at com.google.api.gax.grpc.ChannelPool$ReleasingClientCall$1.onClose(ChannelPool.java:570)
    at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:574)
    at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:72)
    at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:742)
    at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:723)
    at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
    at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    at java.base/java.lang.Thread.run(Thread.java:833)
    Suppressed: com.google.api.gax.rpc.AsyncTaskException: Asynchronous task failed
        at com.google.api.gax.rpc.ApiExceptions.callAndTranslateApiException(ApiExceptions.java:57)
        at com.google.api.gax.rpc.UnaryCallable.call(UnaryCallable.java:112)
        at com.google.cloud.vertexai.generativeai.GenerativeModel.generateContent(GenerativeModel.java:467)
        at com.google.cloud.vertexai.generativeai.GenerativeModel.generateContent(GenerativeModel.java:454)
        at com.google.cloud.vertexai.generativeai.ChatSession.sendMessage(ChatSession.java:247)
        at com.google.cloud.vertexai.generativeai.ChatSession.sendMessage(ChatSession.java:233)
        at com.google.cloud.vertexai.it.ITChatSessionIntegrationTest.sendMessageWithNewConfigs_historyContainsFullConversation(ITChatSessionIntegrationTest.java:159)
        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)