search

cryostatio / cryostat-legacy

OUTDATED - See the new repository below! -- Secure JDK Flight Recorder management for containerized JVMs
https://github.com/cryostatio/cryostat
Other
224 stars 31 forks source link

[Bug] Blocked thread during Rule activation #1498

Open andrewazores opened 1 year ago

andrewazores commented 1 year ago 
WARNING: Thread Thread[vert.x-eventloop-thread-2,5,main] has been blocked for 63367 ms, time limit is 2000 ms
io.vertx.core.VertxException: Thread blocked
        at java.base@17.0.7/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.7/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.7/java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1864)
        at java.base@17.0.7/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.7/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.7/java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1898)
        at java.base@17.0.7/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2072)
        at app[//io.cryostat.net](https://redhat-internal.slack.com//io.cryostat.net).AgentJFRService.getAvailableRecordings(AgentJFRService.java:117)
        at app//io.cryostat.recordings.RecordingArchiveHelper.getDescriptorByName(RecordingArchiveHelper.java:1019)
        at app//io.cryostat.recordings.RecordingArchiveHelper.lambda$saveRecording$1(RecordingArchiveHelper.java:397)
        at app//io.cryostat.recordings.RecordingArchiveHelper$$Lambda$1271/0x0000000801ccd530.execute(Unknown Source)
        at app[//io.cryostat.net](https://redhat-internal.slack.com//io.cryostat.net).TargetConnectionManager.executeConnectedTask(TargetConnectionManager.java:168)
        at app//io.cryostat.recordings.RecordingArchiveHelper.saveRecording(RecordingArchiveHelper.java:393)
        at app//io.cryostat.rules.PeriodicArchiver.performArchival(PeriodicArchiver.java:141)
        at app//io.cryostat.rules.PeriodicArchiver.run(PeriodicArchiver.java:123)
        at app//io.cryostat.rules.RuleProcessor.lambda$activate$5(RuleProcessor.java:323)
        at app//io.cryostat.rules.RuleProcessor$$Lambda$1281/0x0000000801db6000.handle(Unknown Source)
        at app//io.vertx.core.impl.VertxImpl$InternalTimerHandler.handle(VertxImpl.java:927)
        at app//io.vertx.core.impl.VertxImpl$InternalTimerHandler.handle(VertxImpl.java:903)
        at app//io.vertx.core.impl.EventLoopContext.emit(EventLoopContext.java:55)
        at app//io.vertx.core.impl.DuplicatedContext.emit(DuplicatedContext.java:158)
        at app//io.vertx.core.impl.ContextInternal.emit(ContextInternal.java:194)
        at app//io.vertx.core.impl.VertxImpl$InternalTimerHandler.run(VertxImpl.java:921)
        at app//io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98)
        at app//io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:159)
        at app//io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
        at app//io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
        at app//io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
        at app//io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:406)
        at app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.7/java.lang.Thread.run(Thread.java:833)

@aali309 ran into this in smoketest.sh. Not sure exactly how to reproduce it, needs further investigation.

Related to #1448 Possibly related to #1494

andrewazores commented 1 year ago 
WARNING: Thread Thread[vert.x-worker-thread-3,5,JDP Client] has been blocked for 89680 ms, time limit is 60000 ms
io.vertx.core.VertxException: Thread blocked
    at app//io.cryostat.net.TargetConnectionManager.executeConnectedTaskAsync(TargetConnectionManager.java:148)
    at app//io.cryostat.recordings.JvmIdHelper.resolveId(JvmIdHelper.java:140)
    at app//io.cryostat.discovery.DiscoveryStorage.modifyChildrenWithJvmIds(DiscoveryStorage.java:317)
    at app//io.cryostat.discovery.DiscoveryStorage.update(DiscoveryStorage.java:348)
    at app//io.cryostat.discovery.DiscoveryStorage.lambda$start$5(DiscoveryStorage.java:168)
    at app//io.cryostat.discovery.DiscoveryStorage$$Lambda$877/0x000000080183dcf8.onEvent(Unknown Source)
    at app//io.cryostat.util.events.AbstractEventEmitter.lambda$emit$0(AbstractEventEmitter.java:54)
    at app//io.cryostat.util.events.AbstractEventEmitter$$Lambda$1261/0x0000000801a94eb8.accept(Unknown Source)
    at java.base@17.0.7/java.util.concurrent.CopyOnWriteArrayList.forEach(CopyOnWriteArrayList.java:807)
    at java.base@17.0.7/java.util.concurrent.CopyOnWriteArraySet.forEach(CopyOnWriteArraySet.java:425)
    at app//io.cryostat.util.events.AbstractEventEmitter.emit(AbstractEventEmitter.java:52)
    at app//io.cryostat.configuration.CredentialsManager.addCredentials(CredentialsManager.java:167)
    at app//io.cryostat.net.web.http.api.v2.CredentialsPostHandler.handle(CredentialsPostHandler.java:143)
    at app//io.cryostat.net.web.http.api.v2.AbstractV2RequestHandler.handle(AbstractV2RequestHandler.java:108)
    at app//io.cryostat.net.web.http.api.v2.AbstractV2RequestHandler.handle(AbstractV2RequestHandler.java:71)
    at app//io.vertx.ext.web.impl.BlockingHandlerDecorator.lambda$handle$0(BlockingHandlerDecorator.java:48)
    at app//io.vertx.ext.web.impl.BlockingHandlerDecorator$$Lambda$1215/0x0000000801a33768.handle(Unknown Source)
    at app//io.vertx.core.impl.ContextBase.lambda$null$0(ContextBase.java:137)
    at app//io.vertx.core.impl.ContextBase$$Lambda$416/0x0000000801636bd0.handle(Unknown Source)
    at app//io.vertx.core.impl.ContextInternal.dispatch(ContextInternal.java:264)
    at app//io.vertx.core.impl.ContextBase.lambda$executeBlocking$1(ContextBase.java:135)
    at app//io.vertx.core.impl.ContextBase$$Lambda$414/0x0000000801636038.run(Unknown Source)
    at app//io.vertx.core.impl.TaskQueue.run(TaskQueue.java:76)
    at app//io.vertx.core.impl.TaskQueue$$Lambda$199/0x00000008011f1c68.run(Unknown Source)
    at java.base@17.0.7/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
    at java.base@17.0.7/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    at app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    at java.base@17.0.7/java.lang.Thread.run(Thread.java:833)