search

flink-extended / flink-remote-shuffle

Remote Shuffle Service for Flink
Apache License 2.0
191 stars 57 forks source link

ShuffleReadClient is blocked when create channel #77

Closed dangshazi closed 2 years ago

dangshazi commented 2 years ago

Description

We maybe encounter a dead lock

Task stack

"HashJoin(joinType=[LeftOuterJoin], where=[(user_id = user_id0)], select=[user_id, user_id0, tag], build=[right]) [Source: HiveSource-smcdrisk_antifraud.th_dp_whitelist_raw_tab] -> Calc(select=[add_salt(user_id) AS user_id, MyRow(_UTF-16LE'{"fields":[{"user_id":"BIGINT","dp_abtesting_user_tag":"string"}]}', CASE(tag IS NOT NULL, CAST(tag), _UTF-16LE'none':VARCHAR(8) CHARACTER SET "UTF-16LE"), _UTF-16LE'dp_abtesting_user_tag') AS EXPR$1]) -> NotNullEnforcer(fields=[user_id]) -> Sink: Sink(table=[shopee_catalog.fs_database.seamoney_antifraud_buyer_th__dp_abtesting_user_tag_th], fields=[user_id, EXPR$1]) (7/100)#3" #397 prio=5 os_prio=0 tid=0x00007feb2c069800 nid=0xc9eb in Object.wait() [0x00007fea3dad9000]
   java.lang.Thread.State: WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    at java.lang.Object.wait(Object.java:502)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.await(DefaultPromise.java:252)
    - locked <0x00000007ee659160> (a org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap$PendingRegistrationPromise)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.await(DefaultChannelPromise.java:131)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.await(DefaultChannelPromise.java:30)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.sync(DefaultPromise.java:403)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.sync(DefaultChannelPromise.java:119)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.sync(DefaultChannelPromise.java:30)
    at com.alibaba.flink.shuffle.transfer.ConnectionManager.createChannel(ConnectionManager.java:197)
    at com.alibaba.flink.shuffle.transfer.ConnectionManager.getOrCreateChannel(ConnectionManager.java:128)
    at com.alibaba.flink.shuffle.transfer.ConnectionManager.getChannel(ConnectionManager.java:114)
    at com.alibaba.flink.shuffle.transfer.ShuffleReadClient.connect(ShuffleReadClient.java:158)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate.setup(RemoteShuffleInputGate.java:250)
    at org.apache.flink.runtime.taskmanager.InputGateWithMetrics.setup(InputGateWithMetrics.java:84)
    at org.apache.flink.runtime.taskmanager.Task.setupPartitionsAndGates(Task.java:955)
    at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:658)
    at org.apache.flink.runtime.taskmanager.Task.run(Task.java:566)
    at java.lang.Thread.run(Thread.java:748)

"HashJoin(joinType=[LeftOuterJoin], where=[(user_id = user_id0)], select=[user_id, user_id0, tag], build=[right]) [Source: HiveSource-smcdrisk_antifraud.th_dp_whitelist_raw_tab] -> Calc(select=[add_salt(user_id) AS user_id, MyRow(_UTF-16LE'{"fields":[{"user_id":"BIGINT","dp_abtesting_user_tag":"string"}]}', CASE(tag IS NOT NULL, CAST(tag), _UTF-16LE'none':VARCHAR(8) CHARACTER SET "UTF-16LE"), _UTF-16LE'dp_abtesting_user_tag') AS EXPR$1]) -> NotNullEnforcer(fields=[user_id]) -> Sink: Sink(table=[shopee_catalog.fs_database.seamoney_antifraud_buyer_th__dp_abtesting_user_tag_th], fields=[user_id, EXPR$1]) (4/100)#2" #263 prio=5 os_prio=0 tid=0x00007feb2c08c800 nid=0xcded waiting on condition [0x00007fec581eb000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <0x00000007ee75b7b8> (a java.util.concurrent.CompletableFuture$Signaller)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
    at java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1707)
    at java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323)
    at java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1742)
    at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1908)
    at com.alibaba.flink.shuffle.transfer.ConnectionManager.releaseChannel(ConnectionManager.java:165)
    at com.alibaba.flink.shuffle.transfer.ShuffleReadClient.close(ShuffleReadClient.java:276)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate.close(RemoteShuffleInputGate.java:331)
    - locked <0x00000006b40e8898> (a java.lang.Object)
    at org.apache.flink.runtime.taskmanager.InputGateWithMetrics.close(InputGateWithMetrics.java:119)
    at org.apache.flink.runtime.taskmanager.Task.closeAllInputGates(Task.java:1013)
    at org.apache.flink.runtime.taskmanager.Task.releaseResources(Task.java:978)
    at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:904)
    at org.apache.flink.runtime.taskmanager.Task.run(Task.java:566)
    at java.lang.Thread.run(Thread.java:748)

Netty stack

"Remote Shuffle Netty Client (10086)-thread-8" #112 daemon prio=5 os_prio=0 tid=0x00007feb78037000 nid=0x8712 waiting for monitor entry [0x00007fec388fa000]
   java.lang.Thread.State: BLOCKED (on object monitor)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate.onBuffer(RemoteShuffleInputGate.java:466)
    - waiting to lock <0x00000006b40e8898> (a java.lang.Object)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate.lambda$getDataListener$2(RemoteShuffleInputGate.java:498)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate$$Lambda$622/1265568361.accept(Unknown Source)
    at com.alibaba.flink.shuffle.transfer.ShuffleReadClient.dataReceived(ShuffleReadClient.java:206)
    at com.alibaba.flink.shuffle.transfer.ReadClientHandler.channelRead(ReadClientHandler.java:174)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
    at org.apache.flink.shaded.netty4.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
    at com.alibaba.flink.shuffle.transfer.DecoderDelegate.channelRead(DecoderDelegate.java:133)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:792)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:475)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:378)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
    at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    at java.lang.Thread.run(Thread.java:748)

"Remote Shuffle Netty Client (10086)-thread-7" #111 daemon prio=5 os_prio=0 tid=0x00007feb78036000 nid=0x8711 runnable [0x00007fec38bfd000]
   java.lang.Thread.State: RUNNABLE
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait0(Native Method)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native.java:132)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:281)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:351)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
    at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    at java.lang.Thread.run(Thread.java:748)

"Remote Shuffle Netty Client (10086)-thread-6" #110 daemon prio=5 os_prio=0 tid=0x00007feb78035000 nid=0x8710 waiting for monitor entry [0x00007fec38afb000]
   java.lang.Thread.State: BLOCKED (on object monitor)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate.lambda$getFailureListener$3(RemoteShuffleInputGate.java:517)
    - waiting to lock <0x00000006b40e8898> (a java.lang.Object)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate$$Lambda$623/1441833118.accept(Unknown Source)
    at com.alibaba.flink.shuffle.transfer.ShuffleReadClient.channelInactive(ShuffleReadClient.java:226)
    at com.alibaba.flink.shuffle.transfer.ReadClientHandler.channelInactive(ReadClientHandler.java:129)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)
    at org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)
    at org.apache.flink.shaded.netty4.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)
    at org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)
    at com.alibaba.flink.shuffle.transfer.DecoderDelegate.channelInactive(DecoderDelegate.java:114)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:384)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
    at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    at java.lang.Thread.run(Thread.java:748)

"Remote Shuffle Netty Client (10086)-thread-5" #109 daemon prio=5 os_prio=0 tid=0x00007feb78034000 nid=0x870f waiting for monitor entry [0x00007fec584ed000]
   java.lang.Thread.State: BLOCKED (on object monitor)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate.lambda$getFailureListener$3(RemoteShuffleInputGate.java:517)
    - waiting to lock <0x00000006b40e8898> (a java.lang.Object)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate$$Lambda$623/1441833118.accept(Unknown Source)
    at com.alibaba.flink.shuffle.transfer.ShuffleReadClient.channelInactive(ShuffleReadClient.java:226)
    at com.alibaba.flink.shuffle.transfer.ReadClientHandler.channelInactive(ReadClientHandler.java:129)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)
    at org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)
    at org.apache.flink.shaded.netty4.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)
    at org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)
    at com.alibaba.flink.shuffle.transfer.DecoderDelegate.channelInactive(DecoderDelegate.java:114)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:384)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
    at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    at java.lang.Thread.run(Thread.java:748)

"Remote Shuffle Netty Client (10086)-thread-4" #108 daemon prio=5 os_prio=0 tid=0x00007feb78032800 nid=0x870e runnable [0x00007fec386f8000]
   java.lang.Thread.State: RUNNABLE
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native Method)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native.java:148)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native.java:141)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
    at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    at java.lang.Thread.run(Thread.java:748)

"Remote Shuffle Netty Client (10086)-thread-3" #107 daemon prio=5 os_prio=0 tid=0x00007feb78031800 nid=0x870c runnable [0x00007fec53ffe000]
   java.lang.Thread.State: RUNNABLE
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native Method)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native.java:148)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native.java:141)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
    at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    at java.lang.Thread.run(Thread.java:748)

"Remote Shuffle Netty Client (10086)-thread-2" #106 daemon prio=5 os_prio=0 tid=0x00007feb780d0800 nid=0x870b runnable [0x00007fec389fb000]
   java.lang.Thread.State: RUNNABLE
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native Method)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native.java:148)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native.java:141)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
    at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    at java.lang.Thread.run(Thread.java:748)

"Remote Shuffle Netty Client (10086)-thread-1" #101 daemon prio=5 os_prio=0 tid=0x00007feb78005800 nid=0x7d5c waiting for monitor entry [0x00007fec582eb000]
   java.lang.Thread.State: BLOCKED (on object monitor)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate.lambda$getFailureListener$3(RemoteShuffleInputGate.java:517)
    - waiting to lock <0x00000006b40e8898> (a java.lang.Object)
    at com.alibaba.flink.shuffle.plugin.transfer.RemoteShuffleInputGate$$Lambda$623/1441833118.accept(Unknown Source)
    at com.alibaba.flink.shuffle.transfer.ShuffleReadClient.channelInactive(ShuffleReadClient.java:226)
    at com.alibaba.flink.shuffle.transfer.ReadClientHandler.channelInactive(ReadClientHandler.java:129)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)
    at org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)
    at org.apache.flink.shaded.netty4.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)
    at org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)
    at com.alibaba.flink.shuffle.transfer.DecoderDelegate.channelInactive(DecoderDelegate.java:114)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)
    at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901)
    at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
    at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:384)
    at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
    at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    at java.lang.Thread.run(Thread.java:748)
dangshazi commented 2 years ago

io.netty.channel.DefaultChannelPromise#sync is blocked

wsry commented 2 years ago

@dangshazi Thanks for reporting this issue.