springboot环境热重启后客户端再调用报InvokeServerBusyException: Server thread pool busy!

hexiangtao commented 5 years ago

问题描述

版本:

com.alipay.sofa sofaboot-dependencies 3.0.0

org.springframework.boot spring-boot-maven-plugin org.springframework springloaded 1.2.6.RELEASE

修改服务提供方任意代码spring自动热加载，然后服务消费端再调用时报 com.alipay.remoting.rpc.exception.InvokeServerBusyException: Server thread pool busy! the address is 192.168.5.76:12200, id=70 at com.alipay.remoting.rpc.RpcResponseResolver.preProcess(RpcResponseResolver.java:104) ~[bolt-1.4.6.jar:na] at com.alipay.remoting.rpc.RpcResponseResolver.resolveResponseObject(RpcResponseResolver.java:56) ~[bolt-1.4.6.jar:na] at com.alipay.remoting.rpc.RpcRemoting.invokeSync(RpcRemoting.java:186) ~[bolt-1.4.6.jar:na] at com.alipay.remoting.rpc.RpcClientRemoting.invokeSync(RpcClientRemoting.java:67) ~[bolt-1.4.6.jar:na] at com.alipay.remoting.rpc.RpcClient.invokeSync(RpcClient.java:350) ~[bolt-1.4.6.jar:na] at com.alipay.sofa.rpc.transport.bolt.BoltClientTransport.doInvokeSync(BoltClientTransport.java:289) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.transport.bolt.BoltClientTransport.syncSend(BoltClientTransport.java:262) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.client.AbstractCluster.doSendMsg(AbstractCluster.java:509) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.client.AbstractCluster.sendMsg(AbstractCluster.java:480) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.filter.ConsumerInvoker.invoke(ConsumerInvoker.java:60) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.filter.sofatracer.ConsumerTracerFilter.invoke(ConsumerTracerFilter.java:66) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.filter.FilterInvoker.invoke(FilterInvoker.java:96) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.filter.RpcReferenceContextFilter.invoke(RpcReferenceContextFilter.java:80) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.filter.FilterInvoker.invoke(FilterInvoker.java:96) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.filter.ConsumerExceptionFilter.invoke(ConsumerExceptionFilter.java:37) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.filter.FilterInvoker.invoke(FilterInvoker.java:96) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.filter.FilterChain.invoke(FilterChain.java:302) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.client.AbstractCluster.filterChain(AbstractCluster.java:473) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.client.FailoverCluster.doInvoke(FailoverCluster.java:66) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.client.AbstractCluster.invoke(AbstractCluster.java:285) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at com.alipay.sofa.rpc.client.ClientProxyInvoker.invoke(ClientProxyInvoker.java:83) ~[sofa-rpc-all-5.4.7.jar:5.4.7] at io.github.eno.sofarpc.service.HelloWorldService_proxy_0.hello(HelloWorldService_proxy_0.java) ~[na:na] at sun.reflect.GeneratedMethodAccessor44.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_121] at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_121] at com.alipay.sofa.runtime.service.binding.JvmBindingAdapter$JvmServiceInvoker.doInvoke(JvmBindingAdapter.java:200) ~[runtime-sofa-boot-starter-3.0.0.jar:3.0.0] at com.alipay.sofa.runtime.spi.service.ServiceProxy.invoke(ServiceProxy.java:39) ~[runtime-sofa-boot-starter-3.0.0.jar:3.0.0] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:185) ~[spring-aop-5.0.7.RELEASE.jar:5.0.7.RELEASE] at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212) ~[spring-aop-5.0.7.RELEASE.jar:5.0.7.RELEASE] at com.sun.proxy.$Proxy75.hello(Unknown Source) ~[na:na] at io.github.eno.sofarpc.IndexController.index(IndexController.java:20) ~[clas

debug跟踪后发现在 com.alipay.remoting.rpc.protocol。RpcCommandDecoder中178行 command.setResponseStatus(ResponseStatus.valueOf(status)); 正好返回的就是SERVER_THREADPOOL_BUSY值

                        command.setType(type);
                        command.setVersion(ver2);
                        command.setId(requestId);
                        command.setSerializer(serializer);
                        command.setResponseStatus(ResponseStatus.valueOf(status));
                        command.setClazz(clazz);
                        command.setHeader(header);
                        command.setContent(content);
                        command.setResponseTimeMillis(System.currentTimeMillis());
                        command.setResponseHost((InetSocketAddress) ctx.channel()
                            .remoteAddress());