Open martbhell opened 8 years ago
Actually looked back a bit and found more stack traces from the same dcache pool on the 1st of August. They're a bit longer so maybe the last one was truncated.
01 Aug 2016 07:10:39 (csc_fi_43) [door:Xrootd-pacific01:AAU3l8OTNHg Xrootd-pacific01 PoolDeliverFile 0000D460C5E4B5AA45AEAD4DA35718D2BD22] Transfer failed due to a bug java.lang.NullPointerException: null at edu.uchicago.monitor.MonitorChannelHandler.handleResponse(MonitorChannelHandler.java:191) [monitor-7.0.1.jar:na] at edu.uchicago.monitor.MonitorChannelHandler.write(MonitorChannelHandler.java:75) [monitor-7.0.1.jar:na] at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:629) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:686) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:622) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.timeout.IdleStateHandler.write(IdleStateHandler.java:284) [netty-handler-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:629) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:686) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:622) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at org.dcache.xrootd.stream.ChunkedResponseWriteHandler.write(ChunkedResponseWriteHandler.java:89) ~[xrootd4j-3.0.5.jar:3.0.5] at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:629) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:686) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:677) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:703) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at org.dcache.xrootd.core.XrootdRequestHandler.respond(XrootdRequestHandler.java:209) [xrootd4j-3.0.5.jar:3.0.5] at org.dcache.xrootd.core.XrootdRequestHandler.requestReceived(XrootdRequestHandler.java:180) [xrootd4j-3.0.5.jar:3.0.5] at org.dcache.xrootd.core.XrootdRequestHandler.channelRead(XrootdRequestHandler.java:79) [xrootd4j-3.0.5.jar:3.0.5] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:266) [netty-handler-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at edu.uchicago.monitor.MonitorChannelHandler.channelRead(MonitorChannelHandler.java:64) [monitor-7.0.1.jar:na] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:276) [netty-codec-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:263) [netty-codec-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:840) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:511) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) [netty-common-4.0.34.Final.jar:4.0.34.Final] at org.dcache.util.CDCThreadFactory$1.run(CDCThreadFactory.java:53) [dcache-core-2.15.10.jar:2.15.10] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_91] 01 Aug 2016 07:10:39 (csc_fi_43) [door:Xrootd-pacific01:AAU3l8OTNHg Xrootd-pacific01 PoolDeliverFile 0000D460C5E4B5AA45AEAD4DA35718D2BD22] Transfer failed due to a bug java.lang.NullPointerException: null at edu.uchicago.monitor.MonitorChannelHandler.handleResponse(MonitorChannelHandler.java:191) [monitor-7.0.1.jar:na] at edu.uchicago.monitor.MonitorChannelHandler.write(MonitorChannelHandler.java:75) [monitor-7.0.1.jar:na] at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:629) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:686) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:622) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.timeout.IdleStateHandler.write(IdleStateHandler.java:284) [netty-handler-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:629) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:686) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:622) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at org.dcache.xrootd.stream.ChunkedResponseWriteHandler.write(ChunkedResponseWriteHandler.java:89) ~[xrootd4j-3.0.5.jar:3.0.5] at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:629) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:686) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:677) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:703) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at org.dcache.xrootd.core.XrootdRequestHandler.respond(XrootdRequestHandler.java:209) [xrootd4j-3.0.5.jar:3.0.5] at org.dcache.xrootd.core.XrootdRequestHandler.requestReceived(XrootdRequestHandler.java:180) [xrootd4j-3.0.5.jar:3.0.5] at org.dcache.xrootd.core.XrootdRequestHandler.channelRead(XrootdRequestHandler.java:79) [xrootd4j-3.0.5.jar:3.0.5] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:266) [netty-handler-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at edu.uchicago.monitor.MonitorChannelHandler.channelRead(MonitorChannelHandler.java:64) [monitor-7.0.1.jar:na] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:276) [netty-codec-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:263) [netty-codec-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:840) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:511) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) [netty-common-4.0.34.Final.jar:4.0.34.Final] at org.dcache.util.CDCThreadFactory$1.run(CDCThreadFactory.java:53) [dcache-core-2.15.10.jar:2.15.10] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_91] 01 Aug 2016 07:10:39 (csc_fi_43) [door:Xrootd-pacific01:AAU4+shrEmg Xrootd-pacific01 PoolDeliverFile 0000617AD95A76E746A58F19EE098688FEBD] Transfer failed due to a bug java.lang.NullPointerException: null at edu.uchicago.monitor.MonitorChannelHandler.handleResponse(MonitorChannelHandler.java:191) [monitor-7.0.1.jar:na] at edu.uchicago.monitor.MonitorChannelHandler.write(MonitorChannelHandler.java:75) [monitor-7.0.1.jar:na] at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:629) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:686) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:622) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.timeout.IdleStateHandler.write(IdleStateHandler.java:284) [netty-handler-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:629) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:686) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:622) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at org.dcache.xrootd.stream.ChunkedResponseWriteHandler.write(ChunkedResponseWriteHandler.java:89) ~[xrootd4j-3.0.5.jar:3.0.5] at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:629) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:686) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:677) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:703) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at org.dcache.xrootd.core.XrootdRequestHandler.respond(XrootdRequestHandler.java:209) [xrootd4j-3.0.5.jar:3.0.5] at org.dcache.xrootd.core.XrootdRequestHandler.requestReceived(XrootdRequestHandler.java:180) [xrootd4j-3.0.5.jar:3.0.5] at org.dcache.xrootd.core.XrootdRequestHandler.channelRead(XrootdRequestHandler.java:79) [xrootd4j-3.0.5.jar:3.0.5] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:266) [netty-handler-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at edu.uchicago.monitor.MonitorChannelHandler.channelRead(MonitorChannelHandler.java:64) [monitor-7.0.1.jar:na] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:276) [netty-codec-4.0.34.Final.jar:4.0.34.Final] at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:263) [netty-codec-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:307) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:293) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:840) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:511) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354) [netty-transport-4.0.34.Final.jar:4.0.34.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) [netty-common-4.0.34.Final.jar:4.0.34.Final] at org.dcache.util.CDCThreadFactory$1.run(CDCThreadFactory.java:53) [dcache-core-2.15.10.jar:2.15.10] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_91] 01 Aug 2016 07:10:39 (csc_fi_43) [] connection closed before being opened. 01 Aug 2016 07:10:39 (csc_fi_43) [] null
Hello!
Saw an NPE once now during the summer from the plugin. Stack trace below.
openjdk8 centos7 dcache 2.15.10
Own compiled version of the monitoring plugin straight from git ( done at 2016-03-18 ( which is after last commit and the fix for #30 )) because we were seeing way way too many exceptions in the logs.