Closed pdepaepe closed 9 years ago
Please post the exact error message that is printed to the logs of the Graylog node you've sent the request to.
On server.log
2015-07-31T09:23:20.762Z ERROR [AnyExceptionClassMapper] Unhandled exception in REST resource
com.google.common.util.concurrent.UncheckedExecutionException: org.elasticsearch.action.search.SearchPhaseExecutionException: Failed to execute phase [query], all shards failed; shardFailures {[CMF-PliaT62oDaFRUsqw_g][graylog2_5][0]: RemoteTransportException[[Live Wire][inet[/172.17.0.13:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[graylog2_5][0]: from[0],size[1]: Parse Failure [Failed to parse source [{"from":0,"size":1,"query":{"query_string":{"query":":\"\"","allow_leading_wildcard":false}},"post_filter":{"bool":{"must":[{"range":{"timestamp":{"from":"2015-07-31 09:22:20.756","to":"2015-07-31 09:23:20.756","include_lower":true,"include_upper":true}}},{"query":{"query_string":{"query":"streams:55bb3df7e4b0b4bd9eb96912"}}}]}},"sort":[{"timestamp":{"order":"desc"}}]}]]]; nested: QueryParsingException[[graylog2_5] Failed to parse query [:""]]; nested: ParseException[Cannot parse ':""': Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; nested: ParseException[Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; }{[CMF-PliaT62oDaFRUsqw_g][graylog2_5][1]: RemoteTransportException[[Live Wire][inet[/172.17.0.13:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[graylog2_5][1]: from[0],size[1]: Parse Failure [Failed to parse source [{"from":0,"size":1,"query":{"query_string":{"query":":\"\"","allow_leading_wildcard":false}},"post_filter":{"bool":{"must":[{"range":{"timestamp":{"from":"2015-07-31 09:22:20.756","to":"2015-07-31 09:23:20.756","include_lower":true,"include_upper":true}}},{"query":{"query_string":{"query":"streams:55bb3df7e4b0b4bd9eb96912"}}}]}},"sort":[{"timestamp":{"order":"desc"}}]}]]]; nested: QueryParsingException[[graylog2_5] Failed to parse query [:""]]; nested: ParseException[Cannot parse ':""': Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; nested: ParseException[Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; }{[CMF-PliaT62oDaFRUsqw_g][graylog2_5][2]: RemoteTransportException[[Live Wire][inet[/172.17.0.13:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[graylog2_5][2]: from[0],size[1]: Parse Failure [Failed to parse source [{"from":0,"size":1,"query":{"query_string":{"query":":\"\"","allow_leading_wildcard":false}},"post_filter":{"bool":{"must":[{"range":{"timestamp":{"from":"2015-07-31 09:22:20.756","to":"2015-07-31 09:23:20.756","include_lower":true,"include_upper":true}}},{"query":{"query_string":{"query":"streams:55bb3df7e4b0b4bd9eb96912"}}}]}},"sort":[{"timestamp":{"order":"desc"}}]}]]]; nested: QueryParsingException[[graylog2_5] Failed to parse query [:""]]; nested: ParseException[Cannot parse ':""': Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; nested: ParseException[Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; }{[CMF-PliaT62oDaFRUsqw_g][graylog2_5][3]: RemoteTransportException[[Live Wire][inet[/172.17.0.13:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[graylog2_5][3]: from[0],size[1]: Parse Failure [Failed to parse source [{"from":0,"size":1,"query":{"query_string":{"query":":\"\"","allow_leading_wildcard":false}},"post_filter":{"bool":{"must":[{"range":{"timestamp":{"from":"2015-07-31 09:22:20.756","to":"2015-07-31 09:23:20.756","include_lower":true,"include_upper":true}}},{"query":{"query_string":{"query":"streams:55bb3df7e4b0b4bd9eb96912"}}}]}},"sort":[{"timestamp":{"order":"desc"}}]}]]]; nested: QueryParsingException[[graylog2_5] Failed to parse query [:""]]; nested: ParseException[Cannot parse ':""': Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; nested: ParseException[Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; }
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2203)
at com.google.common.cache.LocalCache.get(LocalCache.java:3937)
at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4739)
at org.graylog2.rest.resources.streams.alerts.StreamAlertResource.checkConditions(StreamAlertResource.java:130)
at sun.reflect.GeneratedMethodAccessor90.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:164)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:181)
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$TypeOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:203)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:101)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102)
at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:305)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:288)
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1110)
at org.graylog2.jersey.container.netty.NettyContainer.messageReceived(NettyContainer.java:356)
at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.jboss.netty.handler.execution.ChannelUpstreamEventRunnable.doRun(ChannelUpstreamEventRunnable.java:43)
at org.jboss.netty.handler.execution.ChannelEventRunnable.run(ChannelEventRunnable.java:67)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at org.jboss.netty.handler.execution.MemoryAwareThreadPoolExecutor$MemoryAwareRunnable.run(MemoryAwareThreadPoolExecutor.java:622)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.elasticsearch.action.search.SearchPhaseExecutionException: Failed to execute phase [query], all shards failed; shardFailures {[CMF-PliaT62oDaFRUsqw_g][graylog2_5][0]: RemoteTransportException[[Live Wire][inet[/172.17.0.13:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[graylog2_5][0]: from[0],size[1]: Parse Failure [Failed to parse source [{"from":0,"size":1,"query":{"query_string":{"query":":\"\"","allow_leading_wildcard":false}},"post_filter":{"bool":{"must":[{"range":{"timestamp":{"from":"2015-07-31 09:22:20.756","to":"2015-07-31 09:23:20.756","include_lower":true,"include_upper":true}}},{"query":{"query_string":{"query":"streams:55bb3df7e4b0b4bd9eb96912"}}}]}},"sort":[{"timestamp":{"order":"desc"}}]}]]]; nested: QueryParsingException[[graylog2_5] Failed to parse query [:""]]; nested: ParseException[Cannot parse ':""': Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; nested: ParseException[Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; }{[CMF-PliaT62oDaFRUsqw_g][graylog2_5][1]: RemoteTransportException[[Live Wire][inet[/172.17.0.13:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[graylog2_5][1]: from[0],size[1]: Parse Failure [Failed to parse source [{"from":0,"size":1,"query":{"query_string":{"query":":\"\"","allow_leading_wildcard":false}},"post_filter":{"bool":{"must":[{"range":{"timestamp":{"from":"2015-07-31 09:22:20.756","to":"2015-07-31 09:23:20.756","include_lower":true,"include_upper":true}}},{"query":{"query_string":{"query":"streams:55bb3df7e4b0b4bd9eb96912"}}}]}},"sort":[{"timestamp":{"order":"desc"}}]}]]]; nested: QueryParsingException[[graylog2_5] Failed to parse query [:""]]; nested: ParseException[Cannot parse ':""': Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; nested: ParseException[Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; }{[CMF-PliaT62oDaFRUsqw_g][graylog2_5][2]: RemoteTransportException[[Live Wire][inet[/172.17.0.13:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[graylog2_5][2]: from[0],size[1]: Parse Failure [Failed to parse source [{"from":0,"size":1,"query":{"query_string":{"query":":\"\"","allow_leading_wildcard":false}},"post_filter":{"bool":{"must":[{"range":{"timestamp":{"from":"2015-07-31 09:22:20.756","to":"2015-07-31 09:23:20.756","include_lower":true,"include_upper":true}}},{"query":{"query_string":{"query":"streams:55bb3df7e4b0b4bd9eb96912"}}}]}},"sort":[{"timestamp":{"order":"desc"}}]}]]]; nested: QueryParsingException[[graylog2_5] Failed to parse query [:""]]; nested: ParseException[Cannot parse ':""': Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; nested: ParseException[Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; }{[CMF-PliaT62oDaFRUsqw_g][graylog2_5][3]: RemoteTransportException[[Live Wire][inet[/172.17.0.13:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[graylog2_5][3]: from[0],size[1]: Parse Failure [Failed to parse source [{"from":0,"size":1,"query":{"query_string":{"query":":\"\"","allow_leading_wildcard":false}},"post_filter":{"bool":{"must":[{"range":{"timestamp":{"from":"2015-07-31 09:22:20.756","to":"2015-07-31 09:23:20.756","include_lower":true,"include_upper":true}}},{"query":{"query_string":{"query":"streams:55bb3df7e4b0b4bd9eb96912"}}}]}},"sort":[{"timestamp":{"order":"desc"}}]}]]]; nested: QueryParsingException[[graylog2_5] Failed to parse query [:""]]; nested: ParseException[Cannot parse ':""': Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; nested: ParseException[Encountered " ":" ": "" at line 1, column 0.
Was expecting one of:
<NOT> ...
"+" ...
"-" ...
<BAREOPER> ...
"(" ...
"*" ...
<QUOTED> ...
<TERM> ...
<PREFIXTERM> ...
<WILDTERM> ...
<REGEXPTERM> ...
"[" ...
"{" ...
<NUMBER> ...
<TERM> ...
"*" ...
]; }
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.onFirstPhaseResult(TransportSearchTypeAction.java:238)
at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.onFailure(TransportSearchTypeAction.java:184)
at org.elasticsearch.search.action.SearchServiceTransportAction$6.handleException(SearchServiceTransportAction.java:249)
at org.elasticsearch.transport.netty.MessageChannelHandler.handleException(MessageChannelHandler.java:188)
at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:178)
at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:128)
at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303)
at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
... 3 more
The problem in this case is, that field
and value
are empty and Graylog didn't check this before creating the alert condition. The result is, that the alert scanner thread generates invalid Elasticsearch queries (as the exceptions in your logs show).
Hello,
Adding a (stupid) alert condition on a stream like this:
Crash the stream listing page (Internal 500 errors).