search

futurewei-cloud / alcor

Alcor: Cloud native SDN platform powered by Kubernetes and Istio
MIT License
32 stars 33 forks source link

[PM] I/O error on POST request for IPS #689

Open yanmo96 opened 2 years ago

yanmo96 commented 2 years ago

When doing create list ports 1000 times, 500 rps: http://10.213.43.169:8082/alcor-create-and-delete-ports_1000t-500rps.html#/NeutronNetworks.create_and_list_ports/failures

Bellow are log from PM:

Servlet.service() for servlet [dispatcherServlet] in context with path []
2021-09-14 18:45:09.265 ERROR 1 --- [io-8080-exec-19] o.a.c.c.C.[.[.[/].[dispatcherServlet]    : Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed; nested exception is java.util.concurrent.CompletionException: org.springframework.web.client.ResourceAccessException: I/O error on POST request for "http://ipmanager-service.default.svc.cluster.local:9004/ips": timeout; nested exception is java.net.SocketTimeoutException: timeout] with root cause

java.net.SocketException: Socket closed
    at java.base/java.net.SocketInputStream.read(SocketInputStream.java:183) ~[na:na]
    at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140) ~[na:na]
    at okio.Okio$2.read(Okio.java:140) ~[okio-1.17.2.jar!/:na]
    at okio.AsyncTimeout$2.read(AsyncTimeout.java:237) ~[okio-1.17.2.jar!/:na]
    at okio.RealBufferedSource.indexOf(RealBufferedSource.java:358) ~[okio-1.17.2.jar!/:na]
    at okio.RealBufferedSource.readUtf8LineStrict(RealBufferedSource.java:230) ~[okio-1.17.2.jar!/:na]
    at okhttp3.internal.http1.Http1ExchangeCodec.readHeaderLine(Http1ExchangeCodec.java:242) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http1.Http1ExchangeCodec.readResponseHeaders(Http1ExchangeCodec.java:213) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.connection.Exchange.readResponseHeaders(Exchange.java:115) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.CallServerInterceptor.intercept(CallServerInterceptor.java:94) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:43) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:94) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:88) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:229) ~[okhttp-3.14.7.jar!/:na]
    at okhttp3.RealCall.execute(RealCall.java:81) ~[okhttp-3.14.7.jar!/:na]
    at org.springframework.http.client.OkHttp3ClientHttpRequest.executeInternal(OkHttp3ClientHttpRequest.java:73) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:109) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at io.opentracing.contrib.spring.web.client.TracingRestTemplateInterceptor.intercept(TracingRestTemplateInterceptor.java:89) ~[opentracing-spring-web-2.1.1.jar!/:na]
    at org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:93) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.http.client.InterceptingClientHttpRequest.executeInternal(InterceptingClientHttpRequest.java:77) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:739) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:674) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.web.client.RestTemplate.postForObject(RestTemplate.java:418) ~[spring-web-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at com.futurewei.alcor.web.restclient.IpManagerRestClient.allocateIpAddress(IpManagerRestClient.java:98) ~[web-0.1.0-SNAPSHOT.jar!/:0.1.0-SNAPSHOT]
    at com.futurewei.alcor.web.restclient.IpManagerRestClient$$FastClassBySpringCGLIB$$c991c926.invoke(<generated>) ~[web-0.1.0-SNAPSHOT.jar!/:0.1.0-SNAPSHOT]
    at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) ~[spring-core-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:88) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at com.futurewei.alcor.common.stats.StatisticsAspect.durationStatistics(StatisticsAspect.java:47) ~[common-0.1.0-SNAPSHOT.jar!/:0.1.0-SNAPSHOT]
    at jdk.internal.reflect.GeneratedMethodAccessor71.invoke(Unknown Source) ~[na:na]
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:na]
    at java.base/java.lang.reflect.Method.invoke(Method.java:566) ~[na:na]
    at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:633) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:175) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:95) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691) ~[spring-aop-5.2.5.RELEASE.jar!/:5.2.5.RELEASE]
    at com.futurewei.alcor.web.restclient.IpManagerRestClient$$EnhancerBySpringCGLIB$$961e7bf2.allocateIpAddress(<generated>) ~[web-0.1.0-SNAPSHOT.jar!/:0.1.0-SNAPSHOT]
    at com.futurewei.alcor.portmanager.request.AllocateIpAddressRequest.send(AllocateIpAddressRequest.java:52) ~[classes!/:0.1.0-SNAPSHOT]
    at com.futurewei.alcor.portmanager.request.RequestManager.sendRequest(RequestManager.java:42) ~[classes!/:0.1.0-SNAPSHOT]
    at com.futurewei.alcor.portmanager.request.RequestManager.lambda$sendRequestAsync$0(RequestManager.java:78) ~[classes!/:0.1.0-SNAPSHOT]
    at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1700) ~[na:na]
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[na:na]
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[na:na]
    at java.base/java.lang.Thread.run(Thread.java:834) ~[na:na]