search

AsyncHttpClient / async-http-client

Asynchronous Http and WebSocket Client library for Java
Other
6.29k stars 1.59k forks source link

Chunked request body crash with HTTPS proxy #1571

Closed mallik-soupati closed 6 years ago

mallik-soupati commented 6 years ago

AsyncHttpClient : 2.5.2 Netty : 4.1.28.Final Upstream proxy : Burp 1.7.36

if (proxyConf.isEnabled()) {
                Realm proxyRealm = null;
                if ( !StringUtil.isNullOrEmpty(proxyConf.getProxyPrincipal())) {
                    proxyRealm = new Realm.Builder(proxyConf.getProxyPrincipal(),proxyConf.getProxyPassword())
                        .setScheme(Realm.AuthScheme.BASIC)
                        .setUsePreemptiveAuth(true)
                        .build();
                }
                ProxyServer.Builder proxyBuilder = new ProxyServer.Builder(proxyConf.getProxyHost(), proxyConf.getProxyPort());
                ProxyServer proxyServer = null;
                if ( proxyRealm != null ) {
                    proxyBuilder.setRealm(proxyRealm);
                }
                proxyServer = proxyBuilder
                        .setProxyType(ProxyType.HTTP)
                        .setNonProxyHosts(proxyConf.getProxyNonProxyHosts())
                        .build();
                ahcConfigBuilder.setProxyServer(proxyServer);
            }
            if (!proxyConf.isEnablePeerValidation()) {
                ahcConfigBuilder.setDisableHttpsEndpointIdentificationAlgorithm(true);
            }
}

Above is my code snipped, and I get the below error when execute the request with AsycHttpClient:

2018-08-10 19:12:03,806 ERROR [          AsyncHttpClient-4-16] c.c.c.n.p.flow.WebProxyContext      : Error while processing request
io.netty.handler.codec.UnsupportedMessageTypeException: org.asynchttpclient.netty.request.body.BodyChunkedInput (expected: io.netty.buffer.ByteBuf)
    at io.netty.handler.ssl.SslHandler.write(SslHandler.java:729)
    at io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:738)
    at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:730)
    at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:816)
    at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:723)
    at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.write(CombinedChannelDuplexHandler.java:528)
    at io.netty.handler.codec.MessageToMessageEncoder.write(MessageToMessageEncoder.java:101)
    at io.netty.channel.CombinedChannelDuplexHandler.write(CombinedChannelDuplexHandler.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:738)
    at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:730)
    at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:816)
    at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:723)
    at io.netty.channel.DefaultChannelPipeline.write(DefaultChannelPipeline.java:1061)
    at io.netty.channel.AbstractChannel.write(AbstractChannel.java:295)
    at org.asynchttpclient.netty.request.body.NettyBodyBody.write(NettyBodyBody.java:78)
    at org.asynchttpclient.netty.request.NettyRequestSender.writeRequest(NettyRequestSender.java:426)
    at org.asynchttpclient.netty.request.NettyRequestSender.sendRequestWithOpenChannel(NettyRequestSender.java:259)
    at org.asynchttpclient.netty.request.NettyRequestSender.sendRequestWithCertainForceConnect(NettyRequestSender.java:141)
    at org.asynchttpclient.netty.request.NettyRequestSender.sendRequest(NettyRequestSender.java:113)
    at org.asynchttpclient.netty.request.NettyRequestSender.sendNextRequest(NettyRequestSender.java:548)
    at org.asynchttpclient.netty.request.NettyRequestSender$3.lambda$call$0(NettyRequestSender.java:629)
    at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:511)
    at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:504)
    at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:483)
    at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:424)
    at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:103)
    at io.netty.handler.ssl.SslHandler.setHandshakeSuccess(SslHandler.java:1503)
    at io.netty.handler.ssl.SslHandler.wrapNonAppData(SslHandler.java:915)
    at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1338)
    at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1177)
    at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1221)
    at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
    at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
    at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
    at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
    at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
    at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
    at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
    at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646)
    at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:581)
    at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)
    at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
    at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
    at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    at java.lang.Thread.run(Thread.java:748)

I am not sure if I am missing something. Please let me know if any more information is required further. thanks,

mallik-soupati commented 6 years ago

It was working when used with ahc 1.9.32 (ning) in the same set up.

slandelle commented 6 years ago

It's impossible to figure out your use case based on the information and the code you provided. You provided code with tons of conditional blocks, how could one figure out which part is enabled? Also, could you please provide the request part?

mallik-soupati commented 6 years ago

That was my bad, I would have provided more information. The use case is very basic, was trying to see how the request passes through Proxy. The proxy doesn't expect any credentials. It is a HTTPS proxy (configured with required certificates). I provided below the object states of AsyncHttpConfig and Request objects (copied out from debug session):

Config:
=======
config = {DefaultAsyncHttpClientConfig@14357} 
 followRedirect = false
 maxRedirects = 5
 strict302Handling = false
 compressionEnforced = true
 userAgent = "AHC/2.1"
 realm = null
 maxRequestRetry = 3
 disableUrlEncodingForBoundRequests = false
 useLaxCookieEncoder = false
 disableZeroCopy = false
 keepEncodingHeader = false
 proxyServerSelector = {DefaultAsyncHttpClientConfig$Builder$lambda@14430} 
  arg = {ProxyServer@14440} 
   host = "172.16.17.93"
   port = 8888
   securedPort = 8888
   realm = null
   nonProxyHosts = {Collections$UnmodifiableRandomAccessList@14442}  size = 0
   proxyType = {ProxyType@14443} "HTTP"
 validateResponseHeaders = true
 aggregateWebSocketFrameFragments = true
 enablewebSocketCompression = false
 webSocketMaxBufferSize = 128000000
 webSocketMaxFrameSize = 10240
 connectTimeout = -1
 requestTimeout = -1
 readTimeout = -1
 shutdownQuietPeriod = 2000
 shutdownTimeout = 15000
 keepAlive = true
 pooledConnectionIdleTimeout = -1
 connectionPoolCleanerPeriod = 1000
 connectionTtl = -1
 maxConnections = -1
 maxConnectionsPerHost = -1
 channelPool = null
 connectionSemaphoreFactory = null
 keepAliveStrategy = {AsyncHttpClientPool$AsyncHttpClientPoolProvider$1@14431} 
 useOpenSsl = false
 useInsecureTrustManager = false
 disableHttpsEndpointIdentificationAlgorithm = false
 handshakeTimeout = 10000
 enabledProtocols = {String[3]@14432} 
 enabledCipherSuites = null
 filterInsecureCipherSuites = true
 sslSessionCacheSize = 0
 sslSessionTimeout = 0
 sslContext = null
 sslEngineFactory = null
 requestFilters = {Collections$EmptyList@14395}  size = 0
 responseFilters = {Collections$EmptyList@14395}  size = 0
 ioExceptionFilters = {Collections$EmptyList@14395}  size = 0
 cookieStore = {ThreadSafeCookieStore@14435} 
 threadPoolName = "AsyncHttpClient"
 httpClientCodecMaxInitialLineLength = 4096
 httpClientCodecMaxHeaderSize = 8192
 httpClientCodecMaxChunkSize = 8192
 httpClientCodecInitialBufferSize = 128
 chunkedFileChunkSize = 8192
 channelOptions = {Collections$EmptyMap@14437}  size = 0
 eventLoopGroup = null
 useNativeTransport = false
 allocator = null
 tcpNoDelay = true
 soReuseAddress = false
 soLinger = -1
 soSndBuf = -1
 soRcvBuf = -1
 nettyTimer = null
 threadFactory = null
 httpAdditionalChannelInitializer = null
 wsAdditionalChannelInitializer = null
 responseBodyPartFactory = {AsyncHttpClientConfig$ResponseBodyPartFactory$1@14438} "EAGER"
 ioThreadsCount = 16

Request Object:
===============

request = {DefaultRequest@14388} "https://-----.com/login?loginMethodCookieKey=PWD\tAccept:text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\tUpgrade-Insecure-Requests:1\tUser-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0\tHost:------.com\tAccept-Language:en-US,en;q=0.5\tAccept-Encoding:gzip, deflate\tVia:1.1 csg\tConnection:keep-alive\tContent-Length:0"
 proxyServer = null
 method = "GET"
 uri = {Uri@14392} "https://------.com/login?a=fVtlNTawmp%2fsj0%3d"
 address = null
 localAddress = null
 headers = {DefaultHttpHeaders@14393} "DefaultHttpHeaders[Cookie:loginMethodCookieKey=PWD, Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8, Upgrade-Insecure-Requests: 1, User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0, Host: pmsalesdemo8.successfactors.com, Accept-Language: en-US,en;q=0.5, Accept-Encoding: gzip, deflate, Via: 1.1 csg, Connection: keep-alive, Content-Length: 0]"
 cookies = {ArrayList@14394}  size = 3
 byteData = null
 compositeByteData = null
 stringData = null
 byteBufferData = null
 streamData = null
 bodyGenerator = null
 formParams = {Collections$EmptyList@14395}  size = 0
 bodyParts = {Collections$EmptyList@14395}  size = 0
 virtualHost = null
 realm = null
 file = null
 followRedirect = null
 requestTimeout = 0
 readTimeout = 0
 rangeOffset = 0
 charset = {UTF_8@14396} "UTF-8"
 channelPoolPartitioning = {ChannelPoolPartitioning$PerHostChannelPoolPartitioning@14397} "INSTANCE"
  name = "INSTANCE"
  ordinal = 0
 nameResolver = {DefaultNameResolver@14398} 
 queryParams = null

Please let me know if more information is needed.

mallik-soupati commented 6 years ago

It looks like this exception is seen when POST request is executed, via an upstream proxy. NettyBodyBody.write() creates BodyChunkedInput which is not expected at SSLHandler.write(), as it looks for only ByteBuf.

Is there more information required from me ? to be able to address this issue. Or Am I doing something wrong ? let me know if I need to follow some guidelines to be able to use HTTPS PROXY.

thanks.

mallik-soupati commented 6 years ago

To be able to reproduce this precisely, I modified the following test case of AsyncHttpClient a bit, it produces the exception mentioned.

It appears to be that unless the POST body is NettyDirectBody it will fail in all other cases as well.

org.asynchttpclient.proxy.HttpsProxyTest.testConfigProxy():

@Test
  public void testConfigProxy() throws Exception {
    AsyncHttpClientConfig config = config()
            .setFollowRedirect(false)
            .setKeepAlive(true)
            .setProxyServer(proxyServer("172.16.17.93", 8888).setSecuredPort(8888).setProxyType(ProxyType.HTTP).build())
            .setUseInsecureTrustManager(true)
            .build();
    try (AsyncHttpClient asyncHttpClient = asyncHttpClient(config)) {
      Response r = asyncHttpClient.executeRequest(post(getTargetUrl2()).setBody(new ByteArrayBodyGenerator("POST BODY".getBytes()))).get();
      assertEquals(r.getStatusCode(), 200);
    }
  }

Here is the exception


2018-08-27 16:30:20,396 [AsyncHttpClient-3-1] DEBUG org.asynchttpclient.netty.handler.intercept.ConnectSuccessInterceptor - Connecting to proxy org.asynchttpclient.proxy.ProxyServer@328fe58a for scheme https
2018-08-27 16:30:20,491 [AsyncHttpClient-3-1] DEBUG io.netty.handler.ssl.util.InsecureTrustManagerFactory - Accepting a server certificate: CN=localhost, OU=PortSwigger CA, O=PortSwigger, C=PortSwigger
2018-08-27 16:30:20,581 [AsyncHttpClient-3-1] DEBUG org.asynchttpclient.netty.request.NettyRequestSender - Using open Channel [id: 0xf9923c26, L:/172.16.13.22:58704 - R:/172.16.17.93:8888] for POST '/foo/test'
2018-08-27 16:30:20,591 [AsyncHttpClient-3-1] DEBUG org.asynchttpclient.AsyncCompletionHandler - org.asynchttpclient.netty.request.body.BodyChunkedInput (expected: io.netty.buffer.ByteBuf)
io.netty.handler.codec.UnsupportedMessageTypeException: org.asynchttpclient.netty.request.body.BodyChunkedInput (expected: io.netty.buffer.ByteBuf)
    at io.netty.handler.ssl.SslHandler.write(SslHandler.java:729)
    at io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:738)
    at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:730)
    at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:816)
    at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:723)
    at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.write(CombinedChannelDuplexHandler.java:528)
    at io.netty.handler.codec.MessageToMessageEncoder.write(MessageToMessageEncoder.java:101)
    at io.netty.channel.CombinedChannelDuplexHandler.write(CombinedChannelDuplexHandler.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:738)
    at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:730)
    at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:816)
    at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:723)
    at io.netty.channel.DefaultChannelPipeline.write(DefaultChannelPipeline.java:1061)
    at io.netty.channel.AbstractChannel.write(AbstractChannel.java:295)
    at org.asynchttpclient.netty.request.body.NettyBodyBody.write(NettyBodyBody.java:78)
    at org.asynchttpclient.netty.request.NettyRequestSender.writeRequest(NettyRequestSender.java:426)
    at org.asynchttpclient.netty.request.NettyRequestSender.sendRequestWithOpenChannel(NettyRequestSender.java:259)
    at org.asynchttpclient.netty.request.NettyRequestSender.sendRequestWithCertainForceConnect(NettyRequestSender.java:141)
    at org.asynchttpclient.netty.request.NettyRequestSender.sendRequest(NettyRequestSender.java:113)
    at org.asynchttpclient.netty.request.NettyRequestSender.sendNextRequest(NettyRequestSender.java:548)
    at org.asynchttpclient.netty.request.NettyRequestSender$3.lambda$call$0(NettyRequestSender.java:629)
    at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:511)
    at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:504)
    at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:483)
    at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:424)
    at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:103)
    at io.netty.handler.ssl.SslHandler.setHandshakeSuccess(SslHandler.java:1503)
    at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1346)
    at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1177)
    at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1221)
    at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
    at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
    at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
    at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
    at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
    at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
    at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
    at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646)
    at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:581)
    at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)
    at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
    at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
    at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    at java.lang.Thread.run(Thread.java:748)
2018-08-27 16:30:20,595 [AsyncHttpClient-3-1] DEBUG io.netty.handler.ssl.SslHandler - [id: 0xf9923c26, L:/172.16.13.22:58704 - R:/172.16.17.93:8888] HANDSHAKEN: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
slandelle commented 6 years ago

Thanks for the unit test, it helped!

mallik-soupati commented 6 years ago

When is release 2.5.3 expected ?

mallik-soupati commented 6 years ago

Could you please add it to mile stone 2.5.3 ?

slandelle commented 6 years ago

done today