search

apache / dubbo

The java implementation of Apache Dubbo. An RPC and microservice framework.
https://dubbo.apache.org/
Apache License 2.0
40.55k stars 26.44k forks source link

3.2.1-beta.1/2 provider端只要升级到该版本 ， C端就会无法调用了。报netty解码错误 #11170

Closed mz0113 closed 11 months ago

mz0113 commented 1 year ago

consumer报错截图：

1671512480341

provider报错截图： 1671512424306

consumer端报错详细信息


org.apache.dubbo.rpc.RpcException: Invoke remote method timeout. method: getMetadataInfo, provider: dubbo://192.168.2.104:20881/org.apache.dubbo.metadata.MetadataService?connections=1&corethreads=2&dubbo=2.0.2&group=consumerApp&port=20881&protocol=dubbo&release=3.2.0-beta.1&retries=0&side=provider&threadpool=cached&threads=100&timeout=300&version=1.0.0, cause: org.apache.dubbo.remoting.TimeoutException: Waiting server-side response timeout by scan timer. start time: 2022-12-20 12:58:42.308, end time: 2022-12-20 12:58:42.623, client elapsed: 1 ms, server elapsed: 314 ms, timeout: 300 ms, request: Request [id=2, version=2.0.2, twoWay=true, event=false, broken=false, data=null], channel: /192.168.2.104:60271 -> /192.168.2.104:20881
    at org.apache.dubbo.rpc.protocol.AbstractInvoker.waitForResultIfSync(AbstractInvoker.java:286) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.protocol.AbstractInvoker.invoke(AbstractInvoker.java:180) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.listener.ListenerInvokerWrapper.invoke(ListenerInvokerWrapper.java:70) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at com.bestpay.drip.event.dubbo.filter.ConsumerContextFilter.invoke(ConsumerContextFilter.java:56) ~[event-1.0.3.jar:?]
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:327) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at com.bestpay.drip.ldc.dubbo.AppExFilterConsumer.invoke(AppExFilterConsumer.java:27) ~[ldc-1.0.5-SNAPSHOT.jar:?]
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:327) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CallbackRegistrationInvoker.invoke(FilterChainBuilder.java:194) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.proxy.InvocationUtil.invoke(InvocationUtil.java:55) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.proxy.InvokerInvocationHandler.invoke(InvokerInvocationHandler.java:75) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.metadata.MetadataServiceDubboProxy3.getMetadataInfo(MetadataServiceDubboProxy3.java) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.metadata.MetadataUtils.getRemoteMetadata(MetadataUtils.java:171) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.AbstractServiceDiscovery.getRemoteMetadata(AbstractServiceDiscovery.java:178) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.event.listener.ServiceInstancesChangedListener.lambda$doOnEvent$2(ServiceInstancesChangedListener.java:158) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at java.util.Optional.orElseGet(Optional.java:267) ~[?:1.8.0_292]
    at org.apache.dubbo.registry.client.event.listener.ServiceInstancesChangedListener.doOnEvent(ServiceInstancesChangedListener.java:158) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.event.listener.ServiceInstancesChangedListener.onEvent(ServiceInstancesChangedListener.java:112) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.ServiceDiscoveryRegistry.subscribeURLs(ServiceDiscoveryRegistry.java:316) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.ServiceDiscoveryRegistry.doSubscribe(ServiceDiscoveryRegistry.java:216) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.ServiceDiscoveryRegistry.subscribe(ServiceDiscoveryRegistry.java:184) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.ListenerRegistryWrapper.subscribe(ListenerRegistryWrapper.java:86) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.integration.DynamicDirectory.subscribe(DynamicDirectory.java:183) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.ServiceDiscoveryRegistryDirectory.subscribe(ServiceDiscoveryRegistryDirectory.java:125) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.integration.RegistryProtocol.doCreateInvoker(RegistryProtocol.java:567) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.integration.InterfaceCompatibleRegistryProtocol.getServiceDiscoveryInvoker(InterfaceCompatibleRegistryProtocol.java:65) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.migration.MigrationInvoker.refreshServiceDiscoveryInvoker(MigrationInvoker.java:436) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.migration.MigrationInvoker.migrateToApplicationFirstInvoker(MigrationInvoker.java:244) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.migration.MigrationRuleHandler.refreshInvoker(MigrationRuleHandler.java:73) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.migration.MigrationRuleHandler.doMigrate(MigrationRuleHandler.java:57) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.client.migration.MigrationRuleListener.onRefer(MigrationRuleListener.java:243) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.integration.RegistryProtocol.interceptInvoker(RegistryProtocol.java:532) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.integration.RegistryProtocol.doRefer(RegistryProtocol.java:502) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.registry.integration.RegistryProtocol.refer(RegistryProtocol.java:484) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.protocol.ProtocolListenerWrapper.refer(ProtocolListenerWrapper.java:74) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.qos.protocol.QosProtocolWrapper.refer(QosProtocolWrapper.java:80) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.cluster.filter.ProtocolFilterWrapper.refer(ProtocolFilterWrapper.java:71) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.protocol.ProtocolSerializationWrapper.refer(ProtocolSerializationWrapper.java:52) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.Protocol$Adaptive.refer(Protocol$Adaptive.java) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.config.ReferenceConfig.createInvokerForRemote(ReferenceConfig.java:608) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.config.ReferenceConfig.createProxy(ReferenceConfig.java:446) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.config.ReferenceConfig.init(ReferenceConfig.java:300) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.config.ReferenceConfig.get(ReferenceConfig.java:232) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at com.xxx.test.dubbo.DubboReferenceConfigManager.findGenericService(DubboReferenceConfigManager.java:120) ~[classes/:?]
    at com.xxx.test.dubbo.DubboMessageCallback.invokeDubbo(DubboMessageCallback.java:102) ~[classes/:?]
    at com.xxx.test.dubbo.DubboMessageCallback.lambda$onMessage$11(DubboMessageCallback.java:71) ~[classes/:?]
    at com.xxx.test.dubbo.DubboMessageCallback.callWithSentinel(DubboMessageCallback.java:189) ~[classes/:?]
    at com.xxx.test.dubbo.DubboMessageCallback.onMessage(DubboMessageCallback.java:78) ~[classes/:?]
    at com.xxx.test.consumer.PopMessageConsumer$DefaultMessageListenerConcurrently.consumeMessage(PopMessageConsumer.java:284) ~[classes/:?]
    at org.apache.rocketmq.client.impl.consumer.ConsumeMessagePopConcurrentlyService$ConsumeRequest.run(ConsumeMessagePopConcurrentlyService.java:424) ~[rocketmq-client-5.0.0.jar:5.0.0]
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_292]
    at java.util.concurrent.FutureTask.run$$$capture(FutureTask.java:266) ~[?:1.8.0_292]
    at java.util.concurrent.FutureTask.run(FutureTask.java) ~[?:1.8.0_292]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_292]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_292]
    at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_292]
Caused by: java.util.concurrent.ExecutionException: org.apache.dubbo.remoting.TimeoutException: Waiting server-side response timeout by scan timer. start time: 2022-12-20 12:58:42.308, end time: 2022-12-20 12:58:42.623, client elapsed: 1 ms, server elapsed: 314 ms, timeout: 300 ms, request: Request [id=2, version=2.0.2, twoWay=true, event=false, broken=false, data=null], channel: /192.168.2.104:60271 -> /192.168.2.104:20881
    at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357) ~[?:1.8.0_292]
    at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1928) ~[?:1.8.0_292]
    at org.apache.dubbo.rpc.AsyncRpcResult.get(AsyncRpcResult.java:195) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.protocol.AbstractInvoker.waitForResultIfSync(AbstractInvoker.java:276) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    ... 54 more
Caused by: org.apache.dubbo.remoting.TimeoutException: Waiting server-side response timeout by scan timer. start time: 2022-12-20 12:58:42.308, end time: 2022-12-20 12:58:42.623, client elapsed: 1 ms, server elapsed: 314 ms, timeout: 300 ms, request: Request [id=2, version=2.0.2, twoWay=true, event=false, broken=false, data=null], channel: /192.168.2.104:60271 -> /192.168.2.104:20881
    at org.apache.dubbo.remoting.exchange.support.DefaultFuture.doReceived(DefaultFuture.java:221) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.remoting.exchange.support.DefaultFuture.received(DefaultFuture.java:185) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.remoting.exchange.support.DefaultFuture$TimeoutCheckTask.notifyTimeout(DefaultFuture.java:304) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.remoting.exchange.support.DefaultFuture$TimeoutCheckTask.lambda$run$0(DefaultFuture.java:291) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.common.threadpool.ThreadlessExecutor$RunnableWrapper.run(ThreadlessExecutor.java:184) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.common.threadpool.ThreadlessExecutor.waitAndDrain(ThreadlessExecutor.java:103) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.AsyncRpcResult.get(AsyncRpcResult.java:193) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.protocol.AbstractInvoker.waitForResultIfSync(AbstractInvoker.java:276) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    ... 54 more
[2022-12-20 12:58:44.020][ERROR][com.xxx.test.consumer.PopMessageConsumer 301][ConsumeMessageThread_1][,][null,,,,null,null,null,,,null,null,null,null,{}]messageCallBack.onMessage error  
org.apache.dubbo.rpc.StatusRpcException: CANCELLED : Canceled by remote peer, errorCode=0
RemoteAddress:192.168.2.104:50052
    at org.apache.dubbo.rpc.TriRpcStatus.asException(TriRpcStatus.java:214) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.protocol.tri.DeadlineFuture.doReceived(DeadlineFuture.java:134) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at org.apache.dubbo.rpc.protocol.tri.DeadlineFuture.lambda$received$0(DeadlineFuture.java:91) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_292]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_292]
    at org.apache.dubbo.common.threadlocal.InternalRunnable.run(InternalRunnable.java:41) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_292]

provider端报错详细信息：


io.netty.handler.codec.DecoderException: java.io.IOException: Decode without read data.
    at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:480) ~[netty-codec-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:407) ~[netty-codec-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:374) ~[netty-codec-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:357) ~[netty-codec-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) [netty-transport-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) [netty-transport-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) [netty-transport-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) [netty-transport-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) [netty-transport-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) [netty-transport-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) [netty-transport-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) [netty-transport-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.util.concurrent.AbstractEventExecutor.runTask$$$capture(AbstractEventExecutor.java:174) [netty-common-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java) [netty-common-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.util.concurrent.AbstractEventExecutor.safeExecute$$$capture(AbstractEventExecutor.java:167) [netty-common-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java) [netty-common-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) [netty-common-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500) [netty-transport-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-common-4.1.78.Final.jar:4.1.78.Final]
    at java.lang.Thread.run(Thread.java:748) [na:1.8.0_292]
Caused by: java.io.IOException: Decode without read data.
    at org.apache.dubbo.remoting.transport.netty4.NettyCodecAdapter$InternalDecoder.decode(NettyCodecAdapter.java:92) ~[dubbo-3.2.0-beta.1.jar:3.2.0-beta.1]
    at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[netty-codec-4.1.78.Final.jar:4.1.78.Final]
    at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[netty-codec-4.1.78.Final.jar:4.1.78.Final]
    ... 21 common frames omitted

如图。C端3.0.10或者12或者3.2.1-beta.1/2 都没问题。但是只要P端也从3.0.10或者12 升级到3.2.0beta后拉取元数据时候就报错。

CrazyHZM commented 1 year ago

@mz0113 Is the triple protocol adopted?

mz0113 commented 1 year ago

@CrazyHZM 是triple协议

AlbumenJ commented 1 year ago

@EarthChen PTAL

EarthChen commented 1 year ago

plz provide a demo to reproduce it

mz0113 commented 1 year ago

@EarthChen 


  consumer端：

   //创建dubbo泛化异步引用
    public static synchronized ReferenceConfig<GenericService> createReference(TopicInfoImmutable topicInfoImmutable) {
        ReferenceConfig<GenericService> reference = new ReferenceConfig<>();
        reference.setInterface(topicInfoImmutable.getInterfaceName());
        reference.setGroup(topicInfoImmutable.getGroup());
        reference.setVersion(topicInfoImmutable.getVersion());
        reference.setRetries(1);

        reference.setGeneric(CommonConstants.GENERIC_SERIALIZATION_GSON);
        reference.setAsync(true);
        reference.setCheck(false);
        reference.setFilter("-sentinel.dubbo.consumer.filter");
        //timeout改为在调用时设置
        //reference.setTimeout((int) topicInfoImmutable.getConsumeTimeoutMills());
        return reference;
    }

//执行泛化异步调用 
genericService = createReference（topicInfoImmutable）；
CompletableFuture completableFuture = RpcContext.getContext().asyncCall(
                        () -> genericService.$invoke(info.getMethodName(), info.getArgsType(), args)
                );
                final Object invoke = completableFuture.get();

  //provider端

常规方式，使用@DubboService暴露一个接口
@DubboService
public class MyServiceImpl implements MyService{
   Result doXX();
}

然后C,P端就开始刷那个获取元数据失败的报错了。

AlbumenJ commented 1 year ago

@EarthChen


  consumer端：

   //创建dubbo泛化异步引用
    public static synchronized ReferenceConfig<GenericService> createReference(TopicInfoImmutable topicInfoImmutable) {
        ReferenceConfig<GenericService> reference = new ReferenceConfig<>();
        reference.setInterface(topicInfoImmutable.getInterfaceName());
        reference.setGroup(topicInfoImmutable.getGroup());
        reference.setVersion(topicInfoImmutable.getVersion());
        reference.setRetries(1);

        reference.setGeneric(CommonConstants.GENERIC_SERIALIZATION_GSON);
        reference.setAsync(true);
        reference.setCheck(false);
        reference.setFilter("-sentinel.dubbo.consumer.filter");
        //timeout改为在调用时设置
        //reference.setTimeout((int) topicInfoImmutable.getConsumeTimeoutMills());
        return reference;
    }

//执行泛化异步调用 
genericService = createReference（topicInfoImmutable）；
CompletableFuture completableFuture = RpcContext.getContext().asyncCall(
                        () -> genericService.$invoke(info.getMethodName(), info.getArgsType(), args)
                );
                final Object invoke = completableFuture.get();

  //provider端

常规方式，使用@DubboService暴露一个接口
@DubboService
public class MyServiceImpl implements MyService{
   Result doXX();
}

然后C,P端就开始刷那个获取元数据失败的报错了。

这个我这边没法复现出来，可以提供一个demo嘛

bert82503 commented 6 months ago

我们也遇到这个Caused by: org.apache.dubbo.rpc.StatusRpcException: CANCELLED : Canceled by remote peer, errorCode=1， 使用mesh+proxy-envoy，场景是A-ApiPost7 (HTTPS) -> B (神禹ShenYu网关-dubbo协议) -> C (Dubbo-triple协议) -> D (Dubbo) 【原因】ApiPost7请求设置的content-length头信息引起，一直透传，triple协议调用时没有更新准确的请求内容体长度，引起HTTP头信息冲突和数据读取不完整。

watch org.apache.dubbo.rpc.protocol.tri.TripleInvoker createRequest '{params[1].getAttachments().remove("content-length")}' -x 2
image 
2024-05-29 14:41:38,920 [NettyClientWorker-4-3]  WARN org.apache.dubbo.rpc.protocol.tri.transport.TripleHttp2ClientResponseHandler -  [DUBBO] Triple Client received remote reset errorCode=1, dubbo version: 3.2.10, current host: 192.168.106.86, error code: 4-10. This may be caused by , go to https://dubbo.apache.org/faq/4/10 to find instructions. 
2024-05-29 14:41:38,921 [DubboServerHandler-192.168.106.86:8504-thread-30] ERROR org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CallbackRegistrationInvoker -  [DUBBO] Exception occurred while executing the 1 filter named RpcExceptionFilter., dubbo version: 3.2.10, current host: 192.168.106.86, error code: 2-19. This may be caused by the custom filter is abnormal, go to https://dubbo.apache.org/faq/2/19 to find instructions. 
2024-05-29 14:41:38,922 [DubboServerHandler-192.168.106.86:8504-thread-30] ERROR org.apache.dubbo.rpc.AsyncRpcResult -  [DUBBO] Got exception when trying to fetch the underlying result from AsyncRpcResult., dubbo version: 3.2.10, current host: 192.168.106.86, error code: 3-5. This may be caused by , go to https://dubbo.apache.org/faq/3/5 to find instructions. 
2024-05-29 14:41:38,925 [DubboServerHandler-192.168.106.86:8504-thread-30] ERROR org.apache.dubbo.rpc.AsyncRpcResult -  [DUBBO] Got exception when trying to fetch the underlying result from AsyncRpcResult., dubbo version: 3.2.10, current host: 192.168.106.86, error code: 3-5. This may be caused by , go to https://dubbo.apache.org/faq/3/5 to find instructions. 
2024-05-29 14:41:38,925 [DubboServerHandler-192.168.106.86:8504-thread-30] ERROR com.lefit.dubbo.aop.RequestClientAspect - 请求资源出现问题 method=[getSpuDetail] args=[[AclGetSpuDetailRequest(spuCode=500009589166, skuCode=null, useCache=null)]] response=[null]
org.apache.dubbo.rpc.RpcException: java.util.concurrent.ExecutionException: org.apache.dubbo.rpc.StatusRpcException: CANCELLED : Canceled by remote peer, errorCode=1
    at org.apache.dubbo.rpc.AsyncRpcResult.getAppResponse(AsyncRpcResult.java:180)
    at org.apache.dubbo.rpc.AsyncRpcResult.recreate(AsyncRpcResult.java:245)
    at org.apache.dubbo.rpc.proxy.InvocationUtil.invoke(InvocationUtil.java:64)
    at org.apache.dubbo.rpc.proxy.InvokerInvocationHandler.invoke(InvokerInvocationHandler.java:81)
    at com.leoao.goods.api.SpuSearchApiDubboProxy19.getSpuDetail(SpuSearchApiDubboProxy19.java)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.apache.dubbo.config.spring.util.LazyTargetInvocationHandler.invoke(LazyTargetInvocationHandler.java:54)
    at com.leoao.goods.api.SpuSearchApiDubboProxy19.getSpuDetail(SpuSearchApiDubboProxy19.java)
    at com.lefit.integration.goods.GoodsClientImpl.getSpuDetail(GoodsClientImpl.java:151)
    at com.lefit.integration.goods.GoodsClientImpl$$FastClassBySpringCGLIB$$a11da94c.invoke(<generated>)
    at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
    at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:88)
    at com.lefit.dubbo.aop.RequestClientAspect.requestClient(RequestClientAspect.java:33)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644)
    at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:633)
    at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:175)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
    at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:95)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
    at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691)
    at com.lefit.integration.goods.GoodsClientImpl$$EnhancerBySpringCGLIB$$377a4e34.getSpuDetail(<generated>)
    at com.lefit.dubbo.app.executor.query.SpuQry.getDetail(SpuQry.java:75)
    at com.lefit.dubbo.service.impl.JackyunAppServiceImpl.getSpuDetail(JackyunAppServiceImpl.java:79)
    at com.lefit.dubbo.service.impl.JackyunAppServiceImpl$$FastClassBySpringCGLIB$$c01975a5.invoke(<generated>)
    at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
    at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:88)
    at com.lefit.dubbo.aop.ValidatorAspect.validUser(ValidatorAspect.java:55)
    at sun.reflect.GeneratedMethodAccessor434.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644)
    at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:633)
    at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:175)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
    at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:95)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
    at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691)
    at com.lefit.dubbo.service.impl.JackyunAppServiceImpl$$EnhancerBySpringCGLIB$$b3fb2df.getSpuDetail(<generated>)
    at com.lefit.dubbo.facade.JackyunApiImpl.getSpuDetail(JackyunApiImpl.java:77)
    at com.lefit.dubbo.facade.JackyunApiImpl$$FastClassBySpringCGLIB$$3c586d28.invoke(<generated>)
    at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
    at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:88)
    at com.lefit.dubbo.aop.RequestNotNullAspect.requestNotNull(RequestNotNullAspect.java:46)
    at sun.reflect.GeneratedMethodAccessor433.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644)
    at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:633)
    at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:175)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
    at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:95)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
    at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691)
    at com.lefit.dubbo.facade.JackyunApiImpl$$EnhancerBySpringCGLIB$$3025e348.getSpuDetail(<generated>)
    at com.lefit.dubbo.ec.api.jackyun.JackyunApiDubboWrap19.invokeMethod(JackyunApiDubboWrap19.java)
    at org.apache.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1.doInvoke(JavassistProxyFactory.java:89)
    at org.apache.dubbo.rpc.proxy.AbstractProxyInvoker.invoke(AbstractProxyInvoker.java:100)
    at org.apache.dubbo.config.invoker.DelegateProviderMetaDataInvoker.invoke(DelegateProviderMetaDataInvoker.java:55)
    at org.apache.dubbo.rpc.protocol.InvokerWrapper.invoke(InvokerWrapper.java:56)
    at org.apache.dubbo.rpc.filter.ClassLoaderCallbackFilter.invoke(ClassLoaderCallbackFilter.java:38)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at com.lefit.dubbo.support.filter.ExceptionFilter.invoke(ExceptionFilter.java:36)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at com.leoao.starter.convention.result.ResultWrapperFilter.invoke(ResultWrapperFilter.java:41)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at com.lefit.dubbo.support.filter.DubboAccessLogFilter.invoke(DubboAccessLogFilter.java:95)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.protocol.dubbo.filter.TraceFilter.invoke(TraceFilter.java:80)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.filter.TimeoutFilter.invoke(TimeoutFilter.java:45)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.monitor.support.MonitorFilter.$sw$original$invoke$1vamrv0(MonitorFilter.java:108)
    at org.apache.dubbo.monitor.support.MonitorFilter.$sw$original$invoke$1vamrv0$accessor$$sw$p4vq621(MonitorFilter.java)
    at org.apache.dubbo.monitor.support.MonitorFilter$$sw$auxiliary$1ck1ds2.call(Unknown Source)
    at org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstMethodsInter.intercept(InstMethodsInter.java:86)
    at org.apache.dubbo.monitor.support.MonitorFilter.invoke(MonitorFilter.java)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.filter.ExceptionFilter.invoke(ExceptionFilter.java:54)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.filter.AccessLogFilter.invoke(AccessLogFilter.java:120)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.metrics.observation.ObservationReceiverFilter.invoke(ObservationReceiverFilter.java:57)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.filter.GenericFilter.invoke(GenericFilter.java:217)
    at com.leoao.starter.dubbo.generic.GenericFilterWrapper.invoke(GenericFilterWrapper.java:68)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.filter.ClassLoaderFilter.invoke(ClassLoaderFilter.java:54)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.filter.EchoFilter.invoke(EchoFilter.java:41)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.metrics.filter.MetricsFilter.invoke(MetricsFilter.java:86)
    at org.apache.dubbo.metrics.filter.MetricsProviderFilter.invoke(MetricsProviderFilter.java:37)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.filter.ProfilerServerFilter.invoke(ProfilerServerFilter.java:66)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.filter.ContextFilter.invoke(ContextFilter.java:145)
    at com.leoao.lpaas.compatable.ContextFilterWrapper.invoke(ContextFilterWrapper.java:35)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CallbackRegistrationInvoker.invoke(FilterChainBuilder.java:197)
    at org.apache.dubbo.rpc.protocol.dubbo.DubboProtocol$1.reply(DubboProtocol.java:167)
    at org.apache.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.handleRequest(HeaderExchangeHandler.java:110)
    at org.apache.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.received(HeaderExchangeHandler.java:205)
    at org.apache.dubbo.remoting.transport.DecodeHandler.received(DecodeHandler.java:52)
    at org.apache.dubbo.remoting.transport.dispatcher.ChannelEventRunnable.run(ChannelEventRunnable.java:64)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at org.apache.dubbo.common.threadlocal.InternalRunnable.run(InternalRunnable.java:39)
    at java.lang.Thread.run(Thread.java:750)
Caused by: java.util.concurrent.ExecutionException: org.apache.dubbo.rpc.StatusRpcException: CANCELLED : Canceled by remote peer, errorCode=1
    at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357)
    at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1908)
    at org.apache.dubbo.rpc.AsyncRpcResult.getAppResponse(AsyncRpcResult.java:171)
    ... 129 common frames omitted
Caused by: org.apache.dubbo.rpc.StatusRpcException: CANCELLED : Canceled by remote peer, errorCode=1
    at org.apache.dubbo.rpc.TriRpcStatus.asException(TriRpcStatus.java:213)
    at org.apache.dubbo.rpc.protocol.tri.call.UnaryClientCallListener.onClose(UnaryClientCallListener.java:53)
    at org.apache.dubbo.rpc.protocol.tri.call.TripleClientCall.onComplete(TripleClientCall.java:127)
    at org.apache.dubbo.rpc.protocol.tri.stream.TripleClientStream$ClientTransportListener.finishProcess(TripleClientStream.java:243)
    at org.apache.dubbo.rpc.protocol.tri.stream.TripleClientStream$ClientTransportListener.lambda$cancelByRemote$3(TripleClientStream.java:472)
    at org.apache.dubbo.common.threadpool.serial.SerializingExecutor.run(SerializingExecutor.java:105)
    at org.apache.dubbo.common.threadpool.ThreadlessExecutor$RunnableWrapper.run(ThreadlessExecutor.java:151)
    at org.apache.dubbo.common.threadpool.ThreadlessExecutor.waitAndDrain(ThreadlessExecutor.java:77)
    at org.apache.dubbo.rpc.AsyncRpcResult.get(AsyncRpcResult.java:219)
    at org.apache.dubbo.rpc.protocol.AbstractInvoker.waitForResultIfSync(AbstractInvoker.java:292)
    at org.apache.dubbo.rpc.protocol.AbstractInvoker.invoke(AbstractInvoker.java:194)
    at com.leoao.lpaas.compatable.TripleInvokerWrapper.invoke(TripleInvokerWrapper.java:60)
    at org.apache.dubbo.rpc.listener.ListenerInvokerWrapper.invoke(ListenerInvokerWrapper.java:71)
    at org.apache.dubbo.rpc.filter.RpcExceptionFilter.invoke(RpcExceptionFilter.java:40)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at com.leoao.starter.convention.log.DubboDependencyAbstractLogFilter.invoke(DubboDependencyAbstractLogFilter.java:45)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CallbackRegistrationInvoker.invoke(FilterChainBuilder.java:197)
    at org.apache.dubbo.rpc.protocol.ReferenceCountInvokerWrapper.invoke(ReferenceCountInvokerWrapper.java:106)
    at org.apache.dubbo.rpc.cluster.support.AbstractClusterInvoker.invokeWithContext(AbstractClusterInvoker.java:412)
    at org.apache.dubbo.rpc.cluster.support.FailoverClusterInvoker.doInvoke(FailoverClusterInvoker.java:82)
    at org.apache.dubbo.rpc.cluster.support.AbstractClusterInvoker.invoke(AbstractClusterInvoker.java:366)
    at org.apache.dubbo.rpc.cluster.router.RouterSnapshotFilter.invoke(RouterSnapshotFilter.java:46)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.monitor.support.MonitorFilter.$sw$original$invoke$1vamrv0(MonitorFilter.java:108)
    at org.apache.dubbo.monitor.support.MonitorFilter.$sw$original$invoke$1vamrv0$accessor$$sw$p4vq621(MonitorFilter.java)
    at org.apache.dubbo.monitor.support.MonitorFilter$$sw$auxiliary$1ck1ds2.call(Unknown Source)
    at org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstMethodsInter.intercept(InstMethodsInter.java:86)
    at org.apache.dubbo.monitor.support.MonitorFilter.invoke(MonitorFilter.java)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.cluster.filter.support.MetricsClusterFilter.invoke(MetricsClusterFilter.java:57)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.protocol.dubbo.filter.FutureFilter.invoke(FutureFilter.java:52)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.cluster.filter.support.ObservationSenderFilter.invoke(ObservationSenderFilter.java:62)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.spring.security.filter.ContextHolderParametersSelectedTransferFilter.invoke(ContextHolderParametersSelectedTransferFilter.java:40)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.metrics.filter.MetricsFilter.invoke(MetricsFilter.java:86)
    at org.apache.dubbo.rpc.cluster.filter.support.MetricsConsumerFilter.invoke(MetricsConsumerFilter.java:38)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.cluster.filter.support.ConsumerClassLoaderFilter.invoke(ConsumerClassLoaderFilter.java:40)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.cluster.filter.support.ConsumerContextFilter.invoke(ConsumerContextFilter.java:123)
    at com.leoao.lpaas.compatable.ConsumerContextFilterWrapper.invoke(ConsumerContextFilterWrapper.java:32)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CopyOfFilterChainNode.invoke(FilterChainBuilder.java:349)
    at org.apache.dubbo.rpc.cluster.filter.FilterChainBuilder$CallbackRegistrationInvoker.invoke(FilterChainBuilder.java:197)
    at org.apache.dubbo.rpc.cluster.support.wrapper.AbstractCluster$ClusterFilterInvoker.invoke(AbstractCluster.java:101)
    at org.apache.dubbo.rpc.cluster.support.wrapper.MockClusterInvoker.invoke(MockClusterInvoker.java:106)
    at org.apache.dubbo.rpc.cluster.support.wrapper.ScopeClusterInvoker.invoke(ScopeClusterInvoker.java:156)
    ... 128 common frames omitted