search

sonatype-nexus-community / nexus-repository-apt

A Nexus Repository 3 plugin that allows usage of apt repositories
Eclipse Public License 1.0
105 stars 50 forks source link

Error after upload new package: failed to read header #49

Closed testuser7 closed 6 years ago

testuser7 commented 6 years ago

I'm getting error after upload new package to the repository. Problem is random and not always appears. Nexus version: 3.7.1-02 Plugin version: 1.0.3

Curl command:

curl -u xxx:xxx -X POST -H "Content-Type: multipart/form-data" --data-binary "@name.0.0.1-1+abc.deb" https://nexus/repository/apt/
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100 16.6M    0     0  100 16.6M      0  11.7M  0:00:01  0:00:01 --:--:-- 11.8M
100 16.6M    0     0  100 16.6M      0   9.8M  0:00:01  0:00:01 --:--:--  9.8M
100 16.6M    0     0  100 16.6M      0   9.8M  0:00:01  0:00:01 --:--:--  9.8M```

The problem is that I'm not getting any error message about failed upload from nexus server.

Error in nexus.log:

2018-04-11 08:56:46,794+0200 WARN  [qtp1448346002-48130]  user org.sonatype.nexus.repository.httpbridge.internal.ViewServlet - Failure servicing: POST /repository/apt/
java.io.IOException: failed to read header. Occured at byte: 0
        at org.apache.commons.compress.archivers.ar.ArArchiveInputStream.getNextArEntry(ArArchiveInputStream.java:94)
        at org.apache.commons.compress.archivers.ar.ArArchiveInputStream.getNextEntry(ArArchiveInputStream.java:221)
        at net.staticsnow.nexus.repository.apt.internal.hosted.AptHostedFacet.ingestAsset(AptHostedFacet.java:138)
        at org.sonatype.nexus.transaction.TransactionalWrapper.proceedWithTransaction(TransactionalWrapper.java:56)
        at org.sonatype.nexus.transaction.TransactionInterceptor.invoke(TransactionInterceptor.java:54)
        at net.staticsnow.nexus.repository.apt.internal.hosted.AptHostedHandler.handle(AptHostedHandler.java:63)
        at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
        at net.staticsnow.nexus.repository.apt.internal.gpg.AptSigningHandler.handle(AptSigningHandler.java:47)
        at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
        at net.staticsnow.nexus.repository.apt.internal.snapshot.AptSnapshotHandler.handle(AptSnapshotHandler.java:64)
        at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
        at org.sonatype.nexus.repository.storage.UnitOfWorkHandler.handle(UnitOfWorkHandler.java:39)
        at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
        at org.sonatype.nexus.repository.view.handlers.ContentHeadersHandler.handle(ContentHeadersHandler.java:44)
        at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
        at org.sonatype.nexus.repository.http.PartialFetchHandler.handle(PartialFetchHandler.java:55)
        at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
        at org.sonatype.nexus.repository.view.handlers.ConditionalRequestHandler.handle(ConditionalRequestHandler.java:72)
        at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
        at org.sonatype.nexus.repository.view.handlers.ExceptionHandler.handle(ExceptionHandler.java:44)
        at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
        at org.sonatype.nexus.repository.security.SecurityHandler.handle(SecurityHandler.java:52)
        at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
        at org.sonatype.nexus.repository.view.handlers.TimingHandler.handle(TimingHandler.java:46)
        at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
        at org.sonatype.nexus.repository.view.Context.start(Context.java:114)
        at org.sonatype.nexus.repository.view.Router.dispatch(Router.java:63)
        at org.sonatype.nexus.repository.view.ConfigurableViewFacet.dispatch(ConfigurableViewFacet.java:52)
        at org.sonatype.nexus.repository.view.ConfigurableViewFacet.dispatch(ConfigurableViewFacet.java:43)
        at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.dispatchAndSend(ViewServlet.java:200)
        at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.doService(ViewServlet.java:162)
        at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.service(ViewServlet.java:119)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
        at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:286)
        at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:276)
        at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:181)
        at com.google.inject.servlet.DynamicServletPipeline.service(DynamicServletPipeline.java:71)
        at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85)
        at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:112)
        at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
        at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:61)
        at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108)
        at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137)
        at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
        at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66)
        at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108)
        at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137)
        at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
        at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66)
        at org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449)
        at org.sonatype.nexus.security.SecurityFilter.executeChain(SecurityFilter.java:85)
        at org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365)
        at org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90)
        at org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83)
        at org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383)
        at org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362)
        at org.sonatype.nexus.security.SecurityFilter.doFilterInternal(SecurityFilter.java:101)
        at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
        at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
        at org.sonatype.nexus.repository.httpbridge.internal.ExhaustRequestFilter.doFilter(ExhaustRequestFilter.java:71)
        at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
        at com.sonatype.nexus.licensing.internal.LicensingRedirectFilter.doFilter(LicensingRedirectFilter.java:108)
        at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
        at com.codahale.metrics.servlet.AbstractInstrumentedFilter.doFilter(AbstractInstrumentedFilter.java:97)
        at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
        at org.sonatype.nexus.internal.web.ErrorPageFilter.doFilter(ErrorPageFilter.java:68)
        at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
        at org.sonatype.nexus.internal.web.EnvironmentFilter.doFilter(EnvironmentFilter.java:102)
        at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
        at org.sonatype.nexus.internal.web.HeaderPatternFilter.doFilter(HeaderPatternFilter.java:98)
        at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
        at com.google.inject.servlet.DynamicFilterPipeline.dispatch(DynamicFilterPipeline.java:104)
        at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:135)
        at org.sonatype.nexus.bootstrap.osgi.DelegatingFilter.doFilter(DelegatingFilter.java:73)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1751)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at com.codahale.metrics.jetty9.InstrumentedHandler.handle(InstrumentedHandler.java:175)
        at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
        at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
testuser7 commented 6 years ago

I have found that error is correctly reported:

ip - user [11/Apr/2018:08:56:46 +0200] "POST /repository/apt/ HTTP/1.1" 500 1868 16 "curl/7.35.0"

Curl is always reporting 0 exit code if we don't specify additional parameter: https://curl.haxx.se/docs/manpage.html#-f

mpoindexter commented 6 years ago

@testuser7 are you using the S3 Blob store?

testuser7 commented 6 years ago

@mpoindexter I'm using File Blob Store.

mpoindexter commented 6 years ago

Hmm, maybe a network problem? I've not seen this problem before. I'd suggest seeing if it happens with the latest version of Nexus/the plugin.

mpoindexter commented 6 years ago

I'm going to close this out since as far as I can tell it's not a problem in the code, and it's a very old version of Nexus and the plugin. If it happens on the latest code I'd suggest using Wireshark to troubleshoot if there's a network problem that could be causing the upload to be terminated prematurely.

cmaster11 commented 6 years ago

I'd like to report on this one too, I'm having the same issue with Nexus 3.12.1-01

17:05:26 [Nexus upload] 
 21 65.9M    0     0   21 14.2M      0  52.3M  0:00:01 --:--:--  0:00:01 52.3M
 90 65.9M    0     0   90 59.9M      0  47.1M  0:00:01  0:00:01 --:--:-- 47.1M
100 65.9M    0     0  100 65.9M      0  27.4M  0:00:02  0:00:02 --:--:-- 27.4M
100 65.9M    0     0  100 65.9M      0  19.3M  0:00:03  0:00:03 --:--:-- 19.3M
100 65.9M    0     0  100 65.9M      0  14.9M  0:00:04  0:00:04 --:--:-- 14.9M
100 65.9M    0     0  100 65.9M      0  12.1M  0:00:05  0:00:05 --:--:-- 10.0M
100 65.9M    0     0  100 65.9M      0  10.2M  0:00:06  0:00:06 --:--:-- 1190k
100 65.9M    0     0  100 65.9M      0  9113k  0:00:07  0:00:07 --:--:--     0
100 65.9M    0     0  100 65.9M      0  8029k  0:00:08  0:00:08 --:--:--     0
100 65.9M    0     0  100 65.9M      0  7175k  0:00:09  0:00:09 --:--:--     0
* The requested URL returned error: 500 java.io.IOException: failed to read header. Occured at byte: 0
17:05:34 [Nexus upload] 
100 65.9M    0     0  100 65.9M      0  6997k  0:00:09  0:00:09 --:--:--     0
17:05:34 [Nexus upload] * Closing connection 0
17:05:34 [Nexus upload] curl: (22) The requested URL returned error: 500 java.io.IOException: failed to read header. Occured at byte: 0

It's happening quite frequently, I'm using file storage.

From nexus.log:

2018-07-02 14:05:34,524+0000 WARN  [qtp1098198625-1013]  gocd org.sonatype.nexus.repository.httpbridge.internal.ViewServlet - Failure servicing: POST /repository/hdm-apt-snapshot/
java.io.IOException: failed to read header. Occured at byte: 0
    at org.apache.commons.compress.archivers.ar.ArArchiveInputStream.getNextArEntry(ArArchiveInputStream.java:94)
    at org.apache.commons.compress.archivers.ar.ArArchiveInputStream.getNextEntry(ArArchiveInputStream.java:222)
    at net.staticsnow.nexus.repository.apt.internal.AptPackageParser.parsePackage(AptPackageParser.java:33)
    at net.staticsnow.nexus.repository.apt.internal.hosted.AptHostedFacet.ingestAsset(AptHostedFacet.java:120)
    at org.sonatype.nexus.transaction.TransactionalWrapper.proceedWithTransaction(TransactionalWrapper.java:56)
    at org.sonatype.nexus.transaction.TransactionInterceptor.invoke(TransactionInterceptor.java:54)
    at net.staticsnow.nexus.repository.apt.internal.hosted.AptHostedHandler.handle(AptHostedHandler.java:59)
    at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
    at net.staticsnow.nexus.repository.apt.internal.gpg.AptSigningHandler.handle(AptSigningHandler.java:43)
    at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
    at net.staticsnow.nexus.repository.apt.internal.snapshot.AptSnapshotHandler.handle(AptSnapshotHandler.java:60)
    at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
    at org.sonatype.nexus.repository.storage.UnitOfWorkHandler.handle(UnitOfWorkHandler.java:39)
    at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
    at org.sonatype.nexus.repository.view.handlers.ContentHeadersHandler.handle(ContentHeadersHandler.java:44)
    at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
    at org.sonatype.nexus.repository.http.PartialFetchHandler.handle(PartialFetchHandler.java:55)
    at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
    at org.sonatype.nexus.repository.view.handlers.ConditionalRequestHandler.handle(ConditionalRequestHandler.java:72)
    at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
    at org.sonatype.nexus.repository.view.handlers.ExceptionHandler.handle(ExceptionHandler.java:44)
    at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
    at org.sonatype.nexus.repository.security.SecurityHandler.handle(SecurityHandler.java:52)
    at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
    at org.sonatype.nexus.repository.view.handlers.TimingHandler.handle(TimingHandler.java:46)
    at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
    at org.sonatype.nexus.repository.view.Context.start(Context.java:114)
    at org.sonatype.nexus.repository.view.Router.dispatch(Router.java:64)
    at org.sonatype.nexus.repository.view.ConfigurableViewFacet.dispatch(ConfigurableViewFacet.java:52)
    at org.sonatype.nexus.repository.view.ConfigurableViewFacet.dispatch(ConfigurableViewFacet.java:43)
    at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.dispatchAndSend(ViewServlet.java:210)
    at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.doService(ViewServlet.java:172)
    at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.service(ViewServlet.java:126)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
    at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:286)
    at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:276)
    at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:181)
    at com.google.inject.servlet.DynamicServletPipeline.service(DynamicServletPipeline.java:71)
    at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85)
    at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:112)
    at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
    at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:61)
    at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108)
    at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137)
    at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
    at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66)
    at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108)
    at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137)
    at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
    at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66)
    at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108)
    at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137)
    at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
    at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66)
    at org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449)
    at org.sonatype.nexus.security.SecurityFilter.executeChain(SecurityFilter.java:85)
    at org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365)
    at org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90)
    at org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83)
    at org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383)
    at org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362)
    at org.sonatype.nexus.security.SecurityFilter.doFilterInternal(SecurityFilter.java:101)
    at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
    at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
    at org.sonatype.nexus.repository.httpbridge.internal.ExhaustRequestFilter.doFilter(ExhaustRequestFilter.java:71)
    at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
    at com.sonatype.nexus.licensing.internal.LicensingRedirectFilter.doFilter(LicensingRedirectFilter.java:108)
    at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
    at com.codahale.metrics.servlet.AbstractInstrumentedFilter.doFilter(AbstractInstrumentedFilter.java:97)
    at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
    at org.sonatype.nexus.internal.web.ErrorPageFilter.doFilter(ErrorPageFilter.java:68)
    at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
    at org.sonatype.nexus.internal.web.EnvironmentFilter.doFilter(EnvironmentFilter.java:101)
    at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
    at org.sonatype.nexus.internal.web.HeaderPatternFilter.doFilter(HeaderPatternFilter.java:98)
    at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
    at com.google.inject.servlet.DynamicFilterPipeline.dispatch(DynamicFilterPipeline.java:104)
    at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:135)
    at org.sonatype.nexus.bootstrap.osgi.DelegatingFilter.doFilter(DelegatingFilter.java:73)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1629)
    at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
    at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
    at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
    at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
    at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:190)
    at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
    at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
    at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
    at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
    at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
    at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
    at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
    at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
    at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
    at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
    at com.codahale.metrics.jetty9.InstrumentedHandler.handle(InstrumentedHandler.java:175)
    at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:126)
    at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
    at org.eclipse.jetty.server.Server.handle(Server.java:530)
    at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
    at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
    at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
    at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
    at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
    at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
    at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
    at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
    at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
    at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
    at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
    at java.lang.Thread.run(Thread.java:748)