search

zhouhailin / freeswitch-externals

FreeSWITCH externals projects
https://zhouhailin.github.io/freeswitch-externals/
Apache License 2.0
163 stars 83 forks source link

java.lang.OutOfMemoryError: Direct buffer memory #1

Closed xhuanlee closed 4 years ago

xhuanlee commented 4 years ago

jdk 版本

openjdk version "11" 2018-09-25
OpenJDK Runtime Environment 18.9 (build 11+28)
OpenJDK 64-Bit Server VM 18.9 (build 11+28, mixed mode)

OS

CentOS Linux release 7.5.1804 (Core)

异常

[2019-11-20 07:36:21:766] [nioEventLoopGroup-2-1] [WARN] - io.netty.channel.DefaultChannelPipeline.onUnhandledInboundException(DefaultChannelPipeline.java:1164) - An exceptionCaught() event was fired, and it reached at the tail of the pipeline. It usually means the last handler in the pipeline did not handle the exception.
java.lang.OutOfMemoryError: Direct buffer memory
    at java.nio.Bits.reserveMemory(Bits.java:175) ~[?:?]
    at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:118) ~[?:?]
    at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:317) ~[?:?]
    at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:758) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:734) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:245) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.PoolArena.allocate(PoolArena.java:227) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.PoolArena.allocate(PoolArena.java:147) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:342) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:178) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.AbstractByteBufAllocator.ioBuffer(AbstractByteBufAllocator.java:139) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.DefaultMaxMessagesRecvByteBufAllocator$MaxMessageHandle.allocate(DefaultMaxMessagesRecvByteBufAllocator.java:114) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:147) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:635) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:552) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1044) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at java.lang.Thread.run(Thread.java:834) [?:?]
[2019-11-20 07:36:22:374] [nioEventLoopGroup-2-1] [WARN] - io.netty.channel.DefaultChannelPipeline.onUnhandledInboundException(DefaultChannelPipeline.java:1164) - An exceptionCaught() event was fired, and it reached at the tail of the pipeline. It usually means the last handler in the pipeline did not handle the exception.
java.lang.OutOfMemoryError: Direct buffer memory
    at java.nio.Bits.reserveMemory(Bits.java:175) ~[?:?]
    at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:118) ~[?:?]
    at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:317) ~[?:?]
    at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:758) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:734) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:245) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.PoolArena.allocate(PoolArena.java:227) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.PoolArena.reallocate(PoolArena.java:397) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.PooledByteBuf.capacity(PooledByteBuf.java:119) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.AbstractByteBuf.ensureWritable0(AbstractByteBuf.java:310) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.AbstractByteBuf.ensureWritable(AbstractByteBuf.java:281) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1118) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1111) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1102) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.handler.codec.ByteToMessageDecoder$1.cumulate(ByteToMessageDecoder.java:96) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:281) ~[netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:635) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:552) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1044) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-all-4.1.42.Final.jar!/:4.1.42.Final]
    at java.lang.Thread.run(Thread.java:834) [?:?]
xhuanlee commented 4 years ago

现在JDK换了个版本，正在观察。

java version "1.8.0_231"
Java(TM) SE Runtime Environment (build 1.8.0_231-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.231-b11, mixed mode)
xhuanlee commented 4 years ago

有可能是由于下面异常导致的，没有释放bodyBytes https://github.com/zhouhailin/freeswitch-esl-all/blob/bc76d279c89a717ae209f69aae89f79bc89ebf6a/freeswitch-esl/src/main/java/link/thingscloud/freeswitch/esl/transport/message/EslFrameDecoder.java#L129-L136

[2019-11-20 07:12:54:038] [nioEventLoopGroup-2-1] [ERROR] - io.netty.util.ResourceLeakDetector.reportTracedLeak(ResourceLeakDetector.java:317) - LEAK: ByteBuf.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
#1:
    io.netty.buffer.AdvancedLeakAwareByteBuf.readByte(AdvancedLeakAwareByteBuf.java:400)
    link.thingscloud.freeswitch.esl.transport.message.EslFrameDecoder.readLine(EslFrameDecoder.java:173)
    link.thingscloud.freeswitch.esl.transport.message.EslFrameDecoder.decode(EslFrameDecoder.java:133)
    io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:505)
    io.netty.handler.codec.ReplayingDecoder.callDecode(ReplayingDecoder.java:366)
    io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:283)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
    io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
    io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
    io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931)
    io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
    io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700)
    io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:635)
    io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:552)
    io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514)
    io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1044)
    io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    java.base/java.lang.Thread.run(Thread.java:834)
#2:
    io.netty.buffer.AdvancedLeakAwareByteBuf.writeBytes(AdvancedLeakAwareByteBuf.java:604)
    io.netty.buffer.AbstractByteBuf.readBytes(AbstractByteBuf.java:887)
    io.netty.handler.codec.ReplayingDecoderByteBuf.readBytes(ReplayingDecoderByteBuf.java:577)
    link.thingscloud.freeswitch.esl.transport.message.EslFrameDecoder.decode(EslFrameDecoder.java:129)
    io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:505)
    io.netty.handler.codec.ReplayingDecoder.callDecode(ReplayingDecoder.java:366)
    io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:283)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
    io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
    io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
    io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931)
    io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
    io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700)
    io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:635)
    io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:552)
    io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514)
    io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1044)
    io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    java.base/java.lang.Thread.run(Thread.java:834)
Created at:
    io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:349)
    io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187)
    io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:123)
    io.netty.buffer.AbstractByteBuf.readBytes(AbstractByteBuf.java:886)
    io.netty.handler.codec.ReplayingDecoderByteBuf.readBytes(ReplayingDecoderByteBuf.java:577)
    link.thingscloud.freeswitch.esl.transport.message.EslFrameDecoder.decode(EslFrameDecoder.java:129)
    io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:505)
    io.netty.handler.codec.ReplayingDecoder.callDecode(ReplayingDecoder.java:366)
    io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:283)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
    io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
    io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
    io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
    io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931)
    io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
    io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700)
    io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:635)
    io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:552)
    io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514)
    io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1044)
    io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    java.base/java.lang.Thread.run(Thread.java:834)
: 12 leak records were discarded because they were duplicates
: 2156 leak records were discarded because the leak record count is targeted to 4. Use system property io.netty.leakDetection.targetRecords to increase the limit.
zhouhailin commented 4 years ago

merged in version 1.0.3-BETA1