search

googleapis / google-cloud-java

Google Cloud Client Library for Java
https://cloud.google.com/java/docs/reference
Apache License 2.0
1.89k stars 1.06k forks source link

Flaky Vertex AI: Resource exhausted #11195

Open burkedavison opened 2 weeks ago

burkedavison commented 2 weeks ago

https://fusion2.corp.google.com/invocations/d8831e74-5a7c-47c6-8df3-8b31b049c1da/targets/github%2Fgoogle-cloud-java%2Fjava-vertexai%2Fgoogle-cloud-vertexai%2Ftarget%2Fsurefire-reports;config=default/tests

com.google.api.gax.rpc.ResourceExhaustedException: io.grpc.StatusRuntimeException: RESOURCE_EXHAUSTED: Unable to submit request because the service is temporarily out of capacity. Try again later.
    at com.google.api.gax.rpc.ApiExceptionFactory.createException([ApiExceptionFactory.java:100](https://cs.corp.google.com/#search/&q=f:com/google/api/gax/rpc/ApiExceptionFactory.java:100&ws=&snapshot=0))
    at com.google.api.gax.rpc.ApiExceptionFactory.createException([ApiExceptionFactory.java:41](https://cs.corp.google.com/#search/&q=f:com/google/api/gax/rpc/ApiExceptionFactory.java:41&ws=&snapshot=0))
    at com.google.api.gax.grpc.GrpcApiExceptionFactory.create([GrpcApiExceptionFactory.java:86](https://cs.corp.google.com/#search/&q=f:com/google/api/gax/grpc/GrpcApiExceptionFactory.java:86&ws=&snapshot=0))
    at com.google.api.gax.grpc.GrpcApiExceptionFactory.create([GrpcApiExceptionFactory.java:66](https://cs.corp.google.com/#search/&q=f:com/google/api/gax/grpc/GrpcApiExceptionFactory.java:66&ws=&snapshot=0))
    at com.google.api.gax.grpc.GrpcExceptionCallable$ExceptionTransformingFuture.onFailure([GrpcExceptionCallable.java:97](https://cs.corp.google.com/#search/&q=f:com/google/api/gax/grpc/GrpcExceptionCallable.java:97&ws=&snapshot=0))
    at com.google.api.core.ApiFutures$1.onFailure([ApiFutures.java:84](https://cs.corp.google.com/#search/&q=f:com/google/api/core/ApiFutures.java:84&ws=&snapshot=0))
    at com.google.common.util.concurrent.Futures$CallbackListener.run([Futures.java:1130](https://cs.corp.google.com/#search/&q=f:com/google/common/util/concurrent/Futures.java:1130&ws=&snapshot=0))
    at com.google.common.util.concurrent.DirectExecutor.execute([DirectExecutor.java:31](https://cs.corp.google.com/#search/&q=f:com/google/common/util/concurrent/DirectExecutor.java:31&ws=&snapshot=0))
    at com.google.common.util.concurrent.AbstractFuture.executeListener([AbstractFuture.java:1299](https://cs.corp.google.com/#search/&q=f:com/google/common/util/concurrent/AbstractFuture.java:1299&ws=&snapshot=0))
    at com.google.common.util.concurrent.AbstractFuture.complete([AbstractFuture.java:1060](https://cs.corp.google.com/#search/&q=f:com/google/common/util/concurrent/AbstractFuture.java:1060&ws=&snapshot=0))
    at com.google.common.util.concurrent.AbstractFuture.setException([AbstractFuture.java:810](https://cs.corp.google.com/#search/&q=f:com/google/common/util/concurrent/AbstractFuture.java:810&ws=&snapshot=0))
    at io.grpc.stub.ClientCalls$GrpcFuture.setException(ClientCalls.java:568)
    at io.grpc.stub.ClientCalls$UnaryStreamToFuture.onClose(ClientCalls.java:538)
    at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
    at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
    at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
    at com.google.api.gax.grpc.ChannelPool$ReleasingClientCall$1.onClose([ChannelPool.java:569](https://cs.corp.google.com/#search/&q=f:com/google/api/gax/grpc/ChannelPool.java:569&ws=&snapshot=0))
    at io.grpc.internal.DelayedClientCall$DelayedListener$3.run(DelayedClientCall.java:489)
    at io.grpc.internal.DelayedClientCall$DelayedListener.delayOrExecute(DelayedClientCall.java:453)
    at io.grpc.internal.DelayedClientCall$DelayedListener.onClose(DelayedClientCall.java:486)
    at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:564)
    at io.grpc.internal.ClientCallImpl.access$100(ClientCallImpl.java:72)
    at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:729)
    at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:710)
    at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
    at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker([ThreadPoolExecutor.java:1136](https://cs.corp.google.com/#search/&sq=package:%5Epiper$%20project:openjdk21&q=java/util/concurrent/ThreadPoolExecutor.java:1136&ws=&snapshot=0))
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run([ThreadPoolExecutor.java:635](https://cs.corp.google.com/#search/&sq=package:%5Epiper$%20project:openjdk21&q=java/util/concurrent/ThreadPoolExecutor.java:635%20runWorker&ws=&snapshot=0))
    at java.base/java.lang.Thread.run([Thread.java:833](https://cs.corp.google.com/#search/&sq=package:%5Epiper$%20project:openjdk21&q=java/lang/Thread.java:833%20run&ws=&snapshot=0))
    Suppressed: com.google.api.gax.rpc.AsyncTaskException: Asynchronous task failed
        at com.google.api.gax.rpc.ApiExceptions.callAndTranslateApiException([ApiExceptions.java:57](https://cs.corp.google.com/#search/&q=f:com/google/api/gax/rpc/ApiExceptions.java:57&ws=&snapshot=0))
        at com.google.api.gax.rpc.UnaryCallable.call([UnaryCallable.java:112](https://cs.corp.google.com/#search/&q=f:com/google/api/gax/rpc/UnaryCallable.java:112&ws=&snapshot=0))
        at com.google.cloud.vertexai.generativeai.GenerativeModel.generateContent([GenerativeModel.java:467](https://cs.corp.google.com/#search/&q=f:com/google/cloud/vertexai/generativeai/GenerativeModel.java:467&ws=&snapshot=0))
        at com.google.cloud.vertexai.generativeai.GenerativeModel.generateContent([GenerativeModel.java:454](https://cs.corp.google.com/#search/&q=f:com/google/cloud/vertexai/generativeai/GenerativeModel.java:454&ws=&snapshot=0))
        at com.google.cloud.vertexai.it.ITGenerativeModelIntegrationTest.generateContent_withContentList_nonEmptyCandidate([ITGenerativeModelIntegrationTest.java:200](https://cs.corp.google.com/#search/&q=f:com/google/cloud/vertexai/it/ITGenerativeModelIntegrationTest.java:200&ws=&snapshot=0))
        at java.base/java.lang.reflect.Method.invoke([Method.java:568](https://cs.corp.google.com/#search/&sq=package:%5Epiper$%20project:openjdk21&q=java/lang/reflect/Method.java:568%20generateContent_withContentList_nonEmptyCandidate&ws=&snapshot=0))
Caused by: io.grpc.StatusRuntimeException: RESOURCE_EXHAUSTED: Unable to submit request because the service is temporarily out of capacity. Try again later.
    at io.grpc.Status.asRuntimeException(Status.java:532)
    ... 17 more
lqiu96 commented 1 week ago

This has flaked again in the release PR: https://github.com/googleapis/google-cloud-java/pull/11150