search

Netflix / Hystrix

Hystrix is a latency and fault tolerance library designed to isolate points of access to remote systems, services and 3rd party libraries, stop cascading failure and enable resilience in complex distributed systems where failure is inevitable.
24.09k stars 4.7k forks source link

HystrixSampleSseServlet & nio exceptions #1934

Open IrishkA13 opened 5 years ago

IrishkA13 commented 5 years ago

Faced with randomly but regularly exceptions in logs happened for HystrixSampleSseServlet spring.boot 2.1.2.RELEASE and hystrix 1.5.18

java.nio.BufferOverflowException
    at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:363)
    at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342)
    at sun.nio.ch.IOUtil.write(IOUtil.java:60)
    at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
    at org.apache.tomcat.util.net.NioChannel.write(NioChannel.java:134)
    at org.apache.tomcat.util.net.NioBlockingSelector.write(NioBlockingSelector.java:101)
    at org.apache.tomcat.util.net.NioSelectorPool.write(NioSelectorPool.java:144)
    at org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper.doWrite(NioEndpoint.java:1225)
    at org.apache.tomcat.util.net.SocketWrapperBase.doWrite(SocketWrapperBase.java:743)
    at org.apache.tomcat.util.net.SocketWrapperBase.flushBlocking(SocketWrapperBase.java:696)
    at org.apache.tomcat.util.net.SocketWrapperBase.flush(SocketWrapperBase.java:686)
    at org.apache.coyote.http11.Http11OutputBuffer$SocketOutputBuffer.flush(Http11OutputBuffer.java:553)
    at org.apache.coyote.http11.filters.ChunkedOutputFilter.flush(ChunkedOutputFilter.java:157)
    at org.apache.coyote.http11.Http11OutputBuffer.flush(Http11OutputBuffer.java:216)
    at org.apache.coyote.http11.Http11Processor.flush(Http11Processor.java:1149)
    at org.apache.coyote.AbstractProcessor.action(AbstractProcessor.java:394)
    at org.apache.coyote.Response.action(Response.java:209)
    at org.apache.catalina.connector.OutputBuffer.doFlush(OutputBuffer.java:294)
    at org.apache.catalina.connector.OutputBuffer.flush(OutputBuffer.java:261)
    at org.apache.catalina.connector.CoyoteWriter.flush(CoyoteWriter.java:94)
    at com.netflix.hystrix.contrib.sample.stream.HystrixSampleSseServlet.handleRequest(HystrixSampleSseServlet.java:168)
    at com.netflix.hystrix.contrib.sample.stream.HystrixSampleSseServlet.doGet(HystrixSampleSseServlet.java:74)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:634)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:741)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:199)
    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:490)
    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
    at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:408)
    at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
    at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:834)
    at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1417)
    at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
    at java.lang.Thread.run(Thread.java:748)

    java.nio.InvalidMarkException
        at java.nio.Buffer.reset(Buffer.java:306)
        at org.apache.catalina.connector.OutputBuffer.toReadMode(OutputBuffer.java:831)
        at org.apache.catalina.connector.OutputBuffer.transfer(OutputBuffer.java:805)
        at org.apache.catalina.connector.OutputBuffer.write(OutputBuffer.java:508)
        at org.apache.catalina.connector.CoyoteWriter.write(CoyoteWriter.java:170)
        at org.apache.catalina.connector.CoyoteWriter.write(CoyoteWriter.java:180)
        at org.apache.catalina.connector.CoyoteWriter.print(CoyoteWriter.java:238)
        at com.netflix.hystrix.contrib.sample.stream.HystrixSampleSseServlet.handleRequest(HystrixSampleSseServlet.java:163)
        at com.netflix.hystrix.contrib.sample.stream.HystrixSampleSseServlet.doGet(HystrixSampleSseServlet.java:74)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:634)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:741)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:199)
        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
        at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:490)
        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
        at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:408)
        at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
        at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:834)
        at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1417)
        at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:748)
        java.lang.IllegalArgumentException
            at java.nio.Buffer.position(Buffer.java:244)
            at sun.nio.cs.UTF_8.updatePositions(UTF_8.java:77)
            at sun.nio.cs.UTF_8.access$200(UTF_8.java:57)
            at sun.nio.cs.UTF_8$Encoder.encodeArrayLoop(UTF_8.java:636)
            at sun.nio.cs.UTF_8$Encoder.encodeLoop(UTF_8.java:691)
            at java.nio.charset.CharsetEncoder.encode(CharsetEncoder.java:579)
            at org.apache.tomcat.util.buf.C2BConverter.convert(C2BConverter.java:169)
            at org.apache.catalina.connector.OutputBuffer.realWriteChars(OutputBuffer.java:450)
            at org.apache.catalina.connector.OutputBuffer.flushCharBuffer(OutputBuffer.java:820)
            at org.apache.catalina.connector.OutputBuffer.doFlush(OutputBuffer.java:307)
            at org.apache.catalina.connector.OutputBuffer.flush(OutputBuffer.java:284)
            at org.apache.catalina.connector.CoyoteWriter.flush(CoyoteWriter.java:94)
            at com.netflix.hystrix.contrib.sample.stream.HystrixSampleSseServlet.handleRequest(HystrixSampleSseServlet.java:168)
            at com.netflix.hystrix.contrib.sample.stream.HystrixSampleSseServlet.doGet(HystrixSampleSseServlet.java:74)
            at javax.servlet.http.HttpServlet.service(HttpServlet.java:635)
            at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
            at org.springframework.web.servlet.mvc.ServletWrappingController.handleRequestInternal(ServletWrappingController.java:157)
            at org.springframework.web.servlet.mvc.AbstractController.handleRequest(AbstractController.java:174)
            at org.springframework.cloud.netflix.endpoint.ServletWrappingEndpoint.handle(ServletWrappingEndpoint.java:76)
            at sun.reflect.GeneratedMethodAccessor459.invoke(Unknown Source)
            at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
            at java.lang.reflect.Method.invoke(Method.java:498)
            at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
            at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133)
            at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97)
            at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827)
            at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738)
            at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
            at org.springframework.boot.actuate.autoconfigure.EndpointWebMvcChildContextConfiguration$CompositeHandlerAdapter.handle(EndpointWebMvcChildContextConfiguration.java:286)
            at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)
            at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
            at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
            at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861)
            at javax.servlet.http.HttpServlet.service(HttpServlet.java:635)
            at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
            at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
            at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
            at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
            at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
            at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
            at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:478)
            at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
            at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:80)
            at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
            at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342)
            at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:799)
            at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
            at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
            at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1457)
            at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
            at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
            at java.lang.Thread.run(Thread.java:748)

        java.lang.IllegalArgumentException: Negative capacity: -105
            at java.nio.Buffer.<init>(Buffer.java:199)
            at java.nio.CharBuffer.<init>(CharBuffer.java:281)
            at java.nio.HeapCharBuffer.<init>(HeapCharBuffer.java:86)
            at java.nio.HeapCharBuffer.slice(HeapCharBuffer.java:103)
            at org.apache.catalina.connector.OutputBuffer.flushCharBuffer(OutputBuffer.java:763)
            at org.apache.catalina.connector.OutputBuffer.doFlush(OutputBuffer.java:284)
            at org.apache.catalina.connector.OutputBuffer.flush(OutputBuffer.java:261)
            at org.apache.catalina.connector.CoyoteWriter.flush(CoyoteWriter.java:94)
            at org.apache.catalina.connector.CoyoteWriter.checkError(CoyoteWriter.java:119)
            at com.netflix.hystrix.contrib.sample.stream.HystrixSampleSseServlet.handleRequest(HystrixSampleSseServlet.java:165)
            at com.netflix.hystrix.contrib.sample.stream.HystrixSampleSseServlet.doGet(HystrixSampleSseServlet.java:74)
            at javax.servlet.http.HttpServlet.service(HttpServlet.java:634)
            at javax.servlet.http.HttpServlet.service(HttpServlet.java:741)
            at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
            at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
            at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:199)
            at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
            at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:490)
            at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
            at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
            at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
            at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
            at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:408)
            at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
            at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:834)
            at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1417)
            at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
            at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
            at java.lang.Thread.run(Thread.java:748)
realred commented 5 years ago

me too

Nalha commented 5 years ago

We're getting this but without boot, just a regular spring-web app using hystrix-core 1.5.18.

SEVERE [http-nio-8080-exec-12] org.apache.catalina.core.StandardWrapperValve.invoke Servlet.service() for servlet [HystrixMetricsStreamServlet] in context with path [] threw exception
 java.nio.InvalidMarkException
    at java.nio.Buffer.reset(Buffer.java:306)
    at org.apache.catalina.connector.OutputBuffer.toReadMode(OutputBuffer.java:905)
    at org.apache.catalina.connector.OutputBuffer.transfer(OutputBuffer.java:880)
    at org.apache.catalina.connector.OutputBuffer.write(OutputBuffer.java:539)
    at org.apache.catalina.connector.CoyoteWriter.write(CoyoteWriter.java:170)
    at org.apache.catalina.connector.CoyoteWriter.write(CoyoteWriter.java:180)
    at org.apache.catalina.connector.CoyoteWriter.print(CoyoteWriter.java:238)
    at org.springframework.security.web.util.OnCommittedResponseWrapper$SaveContextPrintWriter.print(OnCommittedResponseWrapper.java:373)
    at org.springframework.security.web.util.OnCommittedResponseWrapper$SaveContextPrintWriter.print(OnCommittedResponseWrapper.java:373)
    at com.netflix.hystrix.contrib.sample.stream.HystrixSampleSseServlet.handleRequest(HystrixSampleSseServlet.java:163)
    at com.netflix.hystrix.contrib.sample.stream.HystrixSampleSseServlet.doGet(HystrixSampleSseServlet.java:74)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:635)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Shoohei commented 5 years ago

I am facing the same issue. The exception is thrown while the "/actuator/hystrix.stream" endpoint is called. Please, give a feedback if you found the cause.

henrik242 commented 5 years ago

Does this patch fix it? https://github.com/Netflix/Hystrix/pull/1757