search

pingcap / tiflash

The analytical engine for TiDB and TiDB Cloud. Try free: https://tidbcloud.com/free-trial
https://docs.pingcap.com/tidb/stable/tiflash-overview
Apache License 2.0
940 stars 409 forks source link

KVStore: Fix AsyncTasksTest.Cancel1 deadlock #8952

Closed CalvinNeo closed 4 months ago

CalvinNeo commented 4 months ago

Bug Report

Please answer these questions before submitting your issue. Thanks!

1. Minimal reproduce step (Required)

This problem is similar to https://github.com/pingcap/tiflash/pull/8848. We should use explicit shutdown.

(gdb) bt
#0  0x00007f1083a2039a in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f1083a252d3 in __pthread_clockjoin_ex () from /lib64/libc.so.6
#2  0x00007f1083e30563 in std::__1::thread::join() () from /usr/local/lib/x86_64-unknown-linux-gnu/libc++.so.1
#3  0x000055af488f56e6 in DB::ThreadPoolImpl<std::__1::thread>::finalize (this=0x7f108286fa00) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:242
#4  0x000055af488f55e9 in DB::ThreadPoolImpl<std::__1::thread>::~ThreadPoolImpl (this=0x7f108286fa00) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:228
#5  0x000055af488f42a9 in DB::GlobalThreadPool::~GlobalThreadPool (this=0x7f108286fa00) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:389
#6  0x000055af4890515b in std::__1::default_delete<DB::GlobalThreadPool>::operator()[abi:v15007](DB::GlobalThreadPool*) const (this=0x55af4a91e250 <DB::GlobalThreadPool::the_instance>, __ptr=0x7f108286fa00) at /usr/local/bin/../include/c++/v1/__memory/unique_ptr.h:48
#7  0x000055af488f878c in std::__1::unique_ptr<DB::GlobalThreadPool, std::__1::default_delete<DB::GlobalThreadPool> >::reset[abi:v15007](DB::GlobalThreadPool*) (this=0x55af4a91e250 <DB::GlobalThreadPool::the_instance>, __p=0x0)
    at /usr/local/bin/../include/c++/v1/__memory/unique_ptr.h:305
#8  0x000055af488f86f9 in std::__1::unique_ptr<DB::GlobalThreadPool, std::__1::default_delete<DB::GlobalThreadPool> >::~unique_ptr[abi:v15007]() (this=0x55af4a91e250 <DB::GlobalThreadPool::the_instance>) at /usr/local/bin/../include/c++/v1/__memory/unique_ptr.h:259
#9  0x00007f10839db455 in __run_exit_handlers () from /lib64/libc.so.6
#10 0x00007f10839db5d0 in exit () from /lib64/libc.so.6
#11 0x00007f10839c3eb7 in __libc_start_call_main () from /lib64/libc.so.6
#12 0x00007f10839c3f60 in __libc_start_main_impl () from /lib64/libc.so.6
#13 0x000055af3bd38de5 in _start ()
(gdb) t 2
[Switching to thread 2 (Thread 0x7f107edff640 (LWP 3482873))]
#0  0x00007f1083a2039a in __futex_abstimed_wait_common () from /lib64/libc.so.6
(gdb) bt
#0  0x00007f1083a2039a in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f1083a22ba0 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f1083e20d6f in std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) () from /usr/local/lib/x86_64-unknown-linux-gnu/libc++.so.1
#3  0x000055af4a04a731 in std::__1::condition_variable::wait<Poco::Event::wait()::$_0>(std::__1::unique_lock<std::__1::mutex>&, Poco::Event::wait()::$_0) (this=0x7f0f6aa0b120, __lk=..., __pred=...) at /usr/local/bin/../include/c++/v1/__mutex_base:398
#4  0x000055af4a04a5ab in Poco::Event::wait (this=0x7f0f6aa0b120) at /DATA/disk1/calvin/tiflash/tics4/contrib/poco/Foundation/src/Event.cpp:79
#5  0x000055af488f82db in DB::ThreadFromGlobalPoolImpl<false>::join (this=0x7f0f6aa0a030) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.h:264
#6  0x000055af488f76a6 in DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::finalize (this=0x7f0f86ad4400) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:242
#7  0x000055af488f75a9 in DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::~ThreadPoolImpl (this=0x7f0f86ad4400) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:228
#8  0x000055af3eba2ffb in std::__1::default_delete<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> > >::operator()[abi:v15007](DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >*) const (this=0x7f0f86ad7e40, __ptr=0x7f0f86ad4400)
    at /usr/local/bin/../include/c++/v1/__memory/unique_ptr.h:48
#9  0x000055af3eba2f7c in std::__1::unique_ptr<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >, std::__1::default_delete<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> > > >::reset[abi:v15007](DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >*)
    (this=0x7f0f86ad7e40, __p=0x0) at /usr/local/bin/../include/c++/v1/__memory/unique_ptr.h:305
#10 0x000055af3eba2a99 in std::__1::unique_ptr<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >, std::__1::default_delete<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> > > >::~unique_ptr[abi:v15007]() (this=0x7f0f86ad7e40)
    at /usr/local/bin/../include/c++/v1/__memory/unique_ptr.h:259
#11 0x000055af4533704b in DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::~AsyncTasks() (this=0x7f0f86ad7e18) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/Utils/AsyncTasks.h:56
#12 0x000055af45336d8d in std::__1::__destroy_at[abi:v15007]<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, 0>(DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>*) (
    __loc=0x7f0f86ad7e18) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:63
#13 0x000055af45336d35 in std::__1::destroy_at[abi:v15007]<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, 0>(DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>*) (
    __loc=0x7f0f86ad7e18) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:88
#14 0x000055af45336d19 in std::__1::allocator_traits<std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> > >::destroy[abi:v15007]<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, void, void>(std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> >&, DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>*) (__p=0x7f0f86ad7e18)
    at /usr/local/bin/../include/c++/v1/__memory/allocator_traits.h:317
#15 0x000055af4533655e in std::__1::__shared_ptr_emplace<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> > >::__on_zero_shared() (this=0x7f0f86ad7e00) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:309
#16 0x000055af3bd44d71 in std::__1::__shared_count::__release_shared[abi:v15007]() (this=0x7f0f86ad7e00) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:174
#17 0x000055af3bd44d19 in std::__1::__shared_weak_count::__release_shared[abi:v15007]() (this=0x7f0f86ad7e00) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:215
--Type <RET> for more, q to quit, c to continue without paging--
#18 0x000055af4533405c in std::__1::shared_ptr<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> >::~shared_ptr[abi:v15007]() (this=0x7f0f869931b8) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:702
#19 0x000055af46d7c186 in DB::FastAddPeerContext::~FastAddPeerContext (this=0x7f0f869931b8) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/MultiRaft/Disagg/FastAddPeerContext.h:33
#20 0x000055af46d7c12d in std::__1::__destroy_at[abi:v15007]<DB::FastAddPeerContext, 0>(DB::FastAddPeerContext*) (__loc=0x7f0f869931b8) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:63
#21 0x000055af46d7c0d5 in std::__1::destroy_at[abi:v15007]<DB::FastAddPeerContext, 0>(DB::FastAddPeerContext*) (__loc=0x7f0f869931b8) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:88
#22 0x000055af46d7c0b9 in std::__1::allocator_traits<std::__1::allocator<DB::FastAddPeerContext> >::destroy[abi:v15007]<DB::FastAddPeerContext, void, void>(std::__1::allocator<DB::FastAddPeerContext>&, DB::FastAddPeerContext*) (__p=0x7f0f869931b8)
    at /usr/local/bin/../include/c++/v1/__memory/allocator_traits.h:317
#23 0x000055af46d7bf5e in std::__1::__shared_ptr_emplace<DB::FastAddPeerContext, std::__1::allocator<DB::FastAddPeerContext> >::__on_zero_shared (this=0x7f0f869931a0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:309

#24 0x000055af3bd44d71 in std::__1::__shared_count::__release_shared[abi:v15007]() (this=0x7f0f869931a0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:174
#25 0x000055af3bd44d19 in std::__1::__shared_weak_count::__release_shared[abi:v15007]() (this=0x7f0f869931a0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:215
#26 0x000055af3ec333cc in std::__1::shared_ptr<DB::FastAddPeerContext>::~shared_ptr[abi:v15007]() (this=0x7f0f6aa000e0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:702
#27 0x000055af45319eb9 in FastAddPeer::$_29::~$_29 (this=0x7f0f6aa000c8) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/MultiRaft/Disagg/FastAddPeer.cpp:631
#28 0x000055af4531c4b5 in std::__1::__compressed_pair_elem<FastAddPeer::$_29, 0, false>::~__compressed_pair_elem (this=0x7f0f6aa000c8) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:30
#29 0x000055af4531c5d5 in std::__1::__compressed_pair<FastAddPeer::$_29, std::__1::allocator<FastAddPeer::$_29> >::~__compressed_pair (this=0x7f0f6aa000c8) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:83
#30 0x000055af4531cd65 in std::__1::__function::__alloc_func<FastAddPeer::$_29, std::__1::allocator<FastAddPeer::$_29>, DB::FastAddPeerRes ()>::destroy[abi:v15007]() (this=0x7f0f6aa000c8) at /usr/local/bin/../include/c++/v1/__functional/function.h:204
#31 0x000055af4531c1a6 in std::__1::__function::__func<FastAddPeer::$_29, std::__1::allocator<FastAddPeer::$_29>, DB::FastAddPeerRes ()>::destroy_deallocate() (this=0x7f0f6aa000c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:351
#32 0x000055af3ec42e21 in std::__1::__function::__value_func<DB::FastAddPeerRes ()>::~__value_func[abi:v15007]() (this=0x7f0f6aa00090) at /usr/local/bin/../include/c++/v1/__functional/function.h:473
#33 0x000055af3ec33875 in std::__1::function<DB::FastAddPeerRes ()>::~function() (this=0x7f0f6aa00090) at /usr/local/bin/../include/c++/v1/__functional/function.h:1184
#34 0x000055af3ec3d395 in std::__1::__compressed_pair_elem<std::__1::function<DB::FastAddPeerRes ()>, 0, false>::~__compressed_pair_elem() (this=0x7f0f6aa00090) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:30
#35 0x000055af3ec3d3b5 in std::__1::__compressed_pair<std::__1::function<DB::FastAddPeerRes ()>, std::__1::allocator<std::__1::function<DB::FastAddPeerRes ()> > >::~__compressed_pair() (this=0x7f0f6aa00090)
    at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:83
#36 0x000055af3ec3d2b6 in std::__1::__packaged_task_func<std::__1::function<DB::FastAddPeerRes ()>, std::__1::allocator<std::__1::function<DB::FastAddPeerRes ()> >, DB::FastAddPeerRes ()>::destroy_deallocate() (this=0x7f0f6aa00080)
    at /usr/local/bin/../include/c++/v1/future:1686
#37 0x000055af3ec3cdfc in std::__1::__packaged_task_function<DB::FastAddPeerRes ()>::~__packaged_task_function() (this=0x7f0f6aa07020) at /usr/local/bin/../include/c++/v1/future:1830
#38 0x000055af3ec3b816 in std::__1::packaged_task<DB::FastAddPeerRes ()>::~packaged_task() (this=0x7f0f6aa07020) at /usr/local/bin/../include/c++/v1/future:1279
#39 0x000055af3ec3cadd in std::__1::__destroy_at[abi:v15007]<std::__1::packaged_task<DB::FastAddPeerRes ()>, 0>(std::__1::packaged_task<DB::FastAddPeerRes ()>*) (__loc=0x7f0f6aa07020) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:63
#40 0x000055af3ec3ca85 in std::__1::destroy_at[abi:v15007]<std::__1::packaged_task<DB::FastAddPeerRes ()>, 0>(std::__1::packaged_task<DB::FastAddPeerRes ()>*) (__loc=0x7f0f6aa07020) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:88
--Type <RET> for more, q to quit, c to continue without paging--
#41 0x000055af3ec3ca69 in std::__1::allocator_traits<std::__1::allocator<std::__1::packaged_task<DB::FastAddPeerRes ()> > >::destroy[abi:v15007]<std::__1::packaged_task<DB::FastAddPeerRes ()>, void, void>(std::__1::allocator<std::__1::packaged_task<DB::FastAddPeerRes ()> >&, std::__1::packaged_task<DB::FastAddPeerRes ()>*) (__p=0x7f0f6aa07020) at /usr/local/bin/../include/c++/v1/__memory/allocator_traits.h:317
#42 0x000055af3ec3c7be in std::__1::__shared_ptr_emplace<std::__1::packaged_task<DB::FastAddPeerRes ()>, std::__1::allocator<std::__1::packaged_task<DB::FastAddPeerRes ()> > >::__on_zero_shared() (this=0x7f0f6aa07000)
    at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:309
#43 0x000055af3bd44d71 in std::__1::__shared_count::__release_shared[abi:v15007]() (this=0x7f0f6aa07000) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:174
#44 0x000055af3bd44d19 in std::__1::__shared_weak_count::__release_shared[abi:v15007]() (this=0x7f0f6aa07000) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:215
#45 0x000055af3ec3baac in std::__1::shared_ptr<std::__1::packaged_task<DB::FastAddPeerRes ()> >::~shared_ptr[abi:v15007]() (this=0x7f0f6aa0b190) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:702
#46 0x000055af3ec3bbfd in DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}::~addTaskWithCancel() (
    this=0x7f0f6aa0b190) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/Utils/AsyncTasks.h:257
#47 0x000055af3ec3ee35 in std::__1::__compressed_pair_elem<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, 0, false>::~__compressed_pair_elem() (this=0x7f0f6aa0b190) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:30
#48 0x000055af3ec3efb5 in std::__1::__compressed_pair<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}> >::~__compressed_pair() (
    this=0x7f0f6aa0b190) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:83

#49 0x000055af3ec3f745 in std::__1::__function::__alloc_func<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}>, void ()>::destroy[abi:v15007]() (this=0x7f0f6aa0b190) at /usr/local/bin/../include/c++/v1/__functional/function.h:204
#50 0x000055af3ec3eb36 in std::__1::__function::__func<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}>, void ()>::destroy_deallocate() (
    this=0x7f0f6aa0b180) at /usr/local/bin/../include/c++/v1/__functional/function.h:351
#51 0x000055af3c2e3ed9 in std::__1::__function::__value_func<void ()>::operator=[abi:v15007](decltype(nullptr)) (this=0x7f107edfa0c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:503

#52 0x000055af3c2e3e61 in std::__1::function<void ()>::operator=(decltype(nullptr)) (this=0x7f107edfa0c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:1170
#53 0x000055af488f7b9a in DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::worker (this=0x7f0f86ad4400, thread_it=...) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:311
#54 0x000055af48904564 in DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}::operator()() const (this=0x7f107edfa1f0)
    at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:160
#55 0x000055af48904535 in std::__1::__invoke[abi:v15007]<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&> (__f=...)
--Type <RET> for more, q to quit, c to continue without paging--
    at /usr/local/bin/../include/c++/v1/__functional/invoke.h:394
#56 0x000055af48904519 in std::__1::__apply_tuple_impl[abi:v15007]<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&, std::__1::__tuple_indices<>) (__f=..., __t=...) at /usr/local/bin/../include/c++/v1/tuple:1789
#57 0x000055af4890449d in std::__1::apply[abi:v15007]<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&) (__f=..., __t=...) at /usr/local/bin/../include/c++/v1/tuple:1798
#58 0x000055af48904413 in DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}::operator()() (this=0x7f0f6aa010c8)
    at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.h:236
#59 0x000055af489043a5 in std::__1::__invoke[abi:v15007]<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}&> (__f=...)
    at /usr/local/bin/../include/c++/v1/__functional/invoke.h:394
#60 0x000055af48904365 in std::__1::__invoke_void_return_wrapper<void, true>::__call<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}&>(DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}&) (__args=...) at /usr/local/bin/../include/c++/v1/__functional/invoke.h:479
#61 0x000055af4890433d in std::__1::__function::__alloc_func<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}, std::__1::allocator<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}>, void ()>::operator()[abi:v15007]() (this=0x7f0f6aa010c8) at /usr/local/bin/../include/c++/v1/__functional/function.h:185
#62 0x000055af48903769 in std::__1::__function::__func<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}, std::__1::allocator<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}>, void ()>::operator()() (this=0x7f0f6aa010c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:359
#63 0x000055af3c2dcdd2 in std::__1::__function::__value_func<void ()>::operator()[abi:v15007]() const (this=0x7f107edfa390) at /usr/local/bin/../include/c++/v1/__functional/function.h:512
#64 0x000055af3c2ccf35 in std::__1::function<void ()>::operator()() const (this=0x7f107edfa390) at /usr/local/bin/../include/c++/v1/__functional/function.h:1197
#65 0x000055af488f5c58 in DB::ThreadPoolImpl<std::__1::thread>::worker (this=0x7f108286fa00, thread_it=...) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:306
--Type <RET> for more, q to quit, c to continue without paging--
#66 0x000055af488fb554 in DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}::operator()() const (this=0x7f1083438ea8)
    at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:160
#67 0x000055af488fb505 in std::__1::__invoke[abi:v15007]<DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}> (__f=...)
    at /usr/local/bin/../include/c++/v1/__functional/invoke.h:394
#68 0x000055af488fb4dd in std::__1::__thread_execute[abi:v15007]<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>&, std::__1::__tuple_indices<>) (__t=...) at /usr/local/bin/../include/c++/v1/thread:284
#69 0x000055af488fb302 in std::__1::__thread_proxy[abi:v15007]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}> >(void*) (__vp=0x7f1083438ea0) at /usr/local/bin/../include/c++/v1/thread:295
#70 0x00007f1083a23802 in start_thread () from /lib64/libc.so.6
#71 0x00007f10839c3450 in clone3 () from /lib64/libc.so.6
(gdb)

2. What did you expect to see? (Required)

3. What did you see instead (Required)

4. What is your TiFlash version? (Required)

CalvinNeo commented 4 months ago 
(gdb) p *(std::__1::mutex*)0x7f0f6aa0b150
$4 = {__m_ = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}}
CalvinNeo commented 4 months ago 
#27 0x000055af45319eb9 in FastAddPeer::$_29::~$_29 (this=0x7f0f6aa000c8) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/MultiRaft/Disagg/FastAddPeer.cpp:631
631             auto job_func = [server, region_id, new_peer_id, fap_ctx, current_time]() {
(gdb) p region_id
$10 = 1
(gdb) p new_peer_id
$11 = 2333
(gdb) p current_time
$12 = 1713270239304
(gdb) p server
$13 = (DB::EngineStoreServerWrap *) 0x7fffcf9e4348
(gdb) p *(DB::EngineStoreServerWrap *)server
$14 = {tmt = 0x0, proxy_helper = 0x7fffcf9e4380, status = {<std::__1::__atomic_base<DB::EngineStoreServerStatus, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<DB::EngineStoreServerStatus>> = {__a_value = 35}, <No data fields>},
      static is_always_lock_free = <optimized out>}, <No data fields>}}
CalvinNeo commented 4 months ago

Destroy of

auto fap_ctx = server->tmt->getContext().getSharedContextDisagg()->fap_context;

dead lock in later lambda and ~FastAddPeerContext.

The lambda is the last owner of fap_context. Its quitting results in the destroy of fap_context, thus AsyncTasks.

main： global thread pool join

worker:

CalvinNeo commented 4 months ago

When the last task finishes in some worker, count() could also be zero.

CalvinNeo commented 4 months ago

Another

(gdb) bt
#0  0x00007fd1e9ab439a in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007fd1e9ab6ba0 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007fd1e9eb4d6f in std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) () from /usr/local/lib/x86_64-unknown-linux-gnu/libc++.so.1
#3  0x000055e50e047c81 in std::__1::condition_variable::wait<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::wait()::{lambda()#1}>(std::__1::unique_lock<std::__1::mutex>&, DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::wait()::{lambda()#1}) (
    this=0x7fd0f24dd528, __lk=..., __pred=...) at /usr/local/bin/../include/c++/v1/__mutex_base:398
#4  0x000055e50e047b7a in DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::wait (this=0x7fd0f24dd500) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:210
#5  0x000055e50aa84463 in DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::shutdown() (this=0x7fd0f2453df8) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/Utils/AsyncTasks.h:59
#6  0x000055e50aa87393 in DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::~AsyncTasks() (this=0x7fd0f2453df8) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/Utils/AsyncTasks.h:67
#7  0x000055e50aa8734d in std::__1::__destroy_at[abi:v15007]<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, 0>(DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>*) (
    __loc=0x7fd0f2453df8) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:63
#8  0x000055e50aa872f5 in std::__1::destroy_at[abi:v15007]<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, 0>(DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>*) (
    __loc=0x7fd0f2453df8) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:88
#9  0x000055e50aa872d9 in std::__1::allocator_traits<std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> > >::destroy[abi:v15007]<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, void, void>(std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> >&, DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>*) (__p=0x7fd0f2453df8)
    at /usr/local/bin/../include/c++/v1/__memory/allocator_traits.h:317
#10 0x000055e50aa86b0e in std::__1::__shared_ptr_emplace<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> > >::__on_zero_shared() (this=0x7fd0f2453de0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:309
#11 0x000055e5014940b1 in std::__1::__shared_count::__release_shared[abi:v15007]() (this=0x7fd0f2453de0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:174
#12 0x000055e501494059 in std::__1::__shared_weak_count::__release_shared[abi:v15007]() (this=0x7fd0f2453de0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:215
#13 0x000055e50aa8414c in std::__1::shared_ptr<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> >::~shared_ptr[abi:v15007]() (this=0x7fd0f24dd428) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:702
#14 0x000055e50c4cc69a in DB::FastAddPeerContext::~FastAddPeerContext (this=0x7fd0f24dd418) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/MultiRaft/Disagg/FastAddPeerContext.h:34
#15 0x000055e50c4cc63d in std::__1::__destroy_at[abi:v15007]<DB::FastAddPeerContext, 0>(DB::FastAddPeerContext*) (__loc=0x7fd0f24dd418) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:63
#16 0x000055e50c4cc5e5 in std::__1::destroy_at[abi:v15007]<DB::FastAddPeerContext, 0>(DB::FastAddPeerContext*) (__loc=0x7fd0f24dd418) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:88
--Type <RET> for more, q to quit, c to continue without paging--
#17 0x000055e50c4cc5c9 in std::__1::allocator_traits<std::__1::allocator<DB::FastAddPeerContext> >::destroy[abi:v15007]<DB::FastAddPeerContext, void, void>(std::__1::allocator<DB::FastAddPeerContext>&, DB::FastAddPeerContext*) (__p=0x7fd0f24dd418)
    at /usr/local/bin/../include/c++/v1/__memory/allocator_traits.h:317
#18 0x000055e50c4cc46e in std::__1::__shared_ptr_emplace<DB::FastAddPeerContext, std::__1::allocator<DB::FastAddPeerContext> >::__on_zero_shared (this=0x7fd0f24dd400) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:309
#19 0x000055e5014940b1 in std::__1::__shared_count::__release_shared[abi:v15007]() (this=0x7fd0f24dd400) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:174
#20 0x000055e501494059 in std::__1::__shared_weak_count::__release_shared[abi:v15007]() (this=0x7fd0f24dd400) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:215
#21 0x000055e504382e8c in std::__1::shared_ptr<DB::FastAddPeerContext>::~shared_ptr[abi:v15007]() (this=0x7fd0d64000e0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:702
#22 0x000055e50aa69f39 in FastAddPeer::$_29::~$_29 (this=0x7fd0d64000c8) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/MultiRaft/Disagg/FastAddPeer.cpp:631
#23 0x000055e50aa6c535 in std::__1::__compressed_pair_elem<FastAddPeer::$_29, 0, false>::~__compressed_pair_elem (this=0x7fd0d64000c8) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:30
#24 0x000055e50aa6c655 in std::__1::__compressed_pair<FastAddPeer::$_29, std::__1::allocator<FastAddPeer::$_29> >::~__compressed_pair (this=0x7fd0d64000c8) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:83
#25 0x000055e50aa6cde5 in std::__1::__function::__alloc_func<FastAddPeer::$_29, std::__1::allocator<FastAddPeer::$_29>, DB::FastAddPeerRes ()>::destroy[abi:v15007]() (this=0x7fd0d64000c8) at /usr/local/bin/../include/c++/v1/__functional/function.h:204
#26 0x000055e50aa6c226 in std::__1::__function::__func<FastAddPeer::$_29, std::__1::allocator<FastAddPeer::$_29>, DB::FastAddPeerRes ()>::destroy_deallocate() (this=0x7fd0d64000c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:351
#27 0x000055e504392ad1 in std::__1::__function::__value_func<DB::FastAddPeerRes ()>::~__value_func[abi:v15007]() (this=0x7fd0d6400090) at /usr/local/bin/../include/c++/v1/__functional/function.h:473
#28 0x000055e504383335 in std::__1::function<DB::FastAddPeerRes ()>::~function() (this=0x7fd0d6400090) at /usr/local/bin/../include/c++/v1/__functional/function.h:1184
#29 0x000055e50438d045 in std::__1::__compressed_pair_elem<std::__1::function<DB::FastAddPeerRes ()>, 0, false>::~__compressed_pair_elem() (this=0x7fd0d6400090) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:30
#30 0x000055e50438d065 in std::__1::__compressed_pair<std::__1::function<DB::FastAddPeerRes ()>, std::__1::allocator<std::__1::function<DB::FastAddPeerRes ()> > >::~__compressed_pair() (this=0x7fd0d6400090)
    at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:83
#31 0x000055e50438cf66 in std::__1::__packaged_task_func<std::__1::function<DB::FastAddPeerRes ()>, std::__1::allocator<std::__1::function<DB::FastAddPeerRes ()> >, DB::FastAddPeerRes ()>::destroy_deallocate() (this=0x7fd0d6400080)
    at /usr/local/bin/../include/c++/v1/future:1686
#32 0x000055e50438caac in std::__1::__packaged_task_function<DB::FastAddPeerRes ()>::~__packaged_task_function() (this=0x7fd0d6407020) at /usr/local/bin/../include/c++/v1/future:1830
#33 0x000055e50438b4c6 in std::__1::packaged_task<DB::FastAddPeerRes ()>::~packaged_task() (this=0x7fd0d6407020) at /usr/local/bin/../include/c++/v1/future:1279
#34 0x000055e50438c78d in std::__1::__destroy_at[abi:v15007]<std::__1::packaged_task<DB::FastAddPeerRes ()>, 0>(std::__1::packaged_task<DB::FastAddPeerRes ()>*) (__loc=0x7fd0d6407020) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:63
#35 0x000055e50438c735 in std::__1::destroy_at[abi:v15007]<std::__1::packaged_task<DB::FastAddPeerRes ()>, 0>(std::__1::packaged_task<DB::FastAddPeerRes ()>*) (__loc=0x7fd0d6407020) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:88
#36 0x000055e50438c719 in std::__1::allocator_traits<std::__1::allocator<std::__1::packaged_task<DB::FastAddPeerRes ()> > >::destroy[abi:v15007]<std::__1::packaged_task<DB::FastAddPeerRes ()>, void, void>(std::__1::allocator<std::__1::packaged_task<DB::FastAddPeerRes ()--Type <RET> for more, q to quit, c to continue without paging--
> >&, std::__1::packaged_task<DB::FastAddPeerRes ()>*) (__p=0x7fd0d6407020) at /usr/local/bin/../include/c++/v1/__memory/allocator_traits.h:317
#37 0x000055e50438c46e in std::__1::__shared_ptr_emplace<std::__1::packaged_task<DB::FastAddPeerRes ()>, std::__1::allocator<std::__1::packaged_task<DB::FastAddPeerRes ()> > >::__on_zero_shared() (this=0x7fd0d6407000)
    at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:309
#38 0x000055e5014940b1 in std::__1::__shared_count::__release_shared[abi:v15007]() (this=0x7fd0d6407000) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:174
#39 0x000055e501494059 in std::__1::__shared_weak_count::__release_shared[abi:v15007]() (this=0x7fd0d6407000) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:215
#40 0x000055e50438b75c in std::__1::shared_ptr<std::__1::packaged_task<DB::FastAddPeerRes ()> >::~shared_ptr[abi:v15007]() (this=0x7fd0d640b190) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:702
#41 0x000055e50438b8ad in DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}::~addTaskWithCancel() (
    this=0x7fd0d640b190) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/Utils/AsyncTasks.h:271
#42 0x000055e50438eae5 in std::__1::__compressed_pair_elem<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, 0, false>::~__compressed_pair_elem() (this=0x7fd0d640b190) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:30
#43 0x000055e50438ec65 in std::__1::__compressed_pair<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}> >::~__compressed_pair() (
    this=0x7fd0d640b190) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:83
#44 0x000055e50438f3f5 in std::__1::__function::__alloc_func<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}>, void ()>::destroy[abi:v15007]() (this=0x7fd0d640b190) at /usr/local/bin/../include/c++/v1/__functional/function.h:204
#45 0x000055e50438e7e6 in std::__1::__function::__func<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}>, void ()>::destroy_deallocate() (
    this=0x7fd0d640b180) at /usr/local/bin/../include/c++/v1/__functional/function.h:351
#46 0x000055e501a33219 in std::__1::__function::__value_func<void ()>::operator=[abi:v15007](decltype(nullptr)) (this=0x7fd1e51fa0c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:503
#47 0x000055e501a331a1 in std::__1::function<void ()>::operator=(decltype(nullptr)) (this=0x7fd1e51fa0c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:1170
#48 0x000055e50e04829a in DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::worker (this=0x7fd0f24dd500, thread_it=...) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:311
#49 0x000055e50e054c64 in DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}::operator()() const (this=0x7fd1e51fa1f0)
--Type <RET> for more, q to quit, c to continue without paging--
    at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:160
#50 0x000055e50e054c35 in std::__1::__invoke[abi:v15007]<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&> (__f=...)
    at /usr/local/bin/../include/c++/v1/__functional/invoke.h:394
#51 0x000055e50e054c19 in std::__1::__apply_tuple_impl[abi:v15007]<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&, std::__1::__tuple_indices<>) (__f=..., __t=...) at /usr/local/bin/../include/c++/v1/tuple:1789
#52 0x000055e50e054b9d in std::__1::apply[abi:v15007]<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&) (__f=..., __t=...) at /usr/local/bin/../include/c++/v1/tuple:1798
#53 0x000055e50e054b13 in DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}::operator()() (this=0x7fd0d64010c8)
    at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.h:236
#54 0x000055e50e054aa5 in std::__1::__invoke[abi:v15007]<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}&> (__f=...)
    at /usr/local/bin/../include/c++/v1/__functional/invoke.h:394
#55 0x000055e50e054a65 in std::__1::__invoke_void_return_wrapper<void, true>::__call<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}&>(DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}&) (__args=...) at /usr/local/bin/../include/c++/v1/__functional/invoke.h:479
#56 0x000055e50e054a3d in std::__1::__function::__alloc_func<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}, std::__1::allocator<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}>, void ()>::operator()[abi:v15007]() (this=0x7fd0d64010c8) at /usr/local/bin/../include/c++/v1/__functional/function.h:185
#57 0x000055e50e053e69 in std::__1::__function::__func<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}, std::__1::allocator<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlo--Type <RET> for more, q to quit, c to continue without paging--
balPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}>, void ()>::operator()() (this=0x7fd0d64010c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:359
#58 0x000055e501a2c112 in std::__1::__function::__value_func<void ()>::operator()[abi:v15007]() const (this=0x7fd1e51fa390) at /usr/local/bin/../include/c++/v1/__functional/function.h:512
#59 0x000055e501a1c275 in std::__1::function<void ()>::operator()() const (this=0x7fd1e51fa390) at /usr/local/bin/../include/c++/v1/__functional/function.h:1197
#60 0x000055e50e046358 in DB::ThreadPoolImpl<std::__1::thread>::worker (this=0x7fd1e8a6fa00, thread_it=...) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:306
#61 0x000055e50e04bc54 in DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}::operator()() const (this=0x7fd1e9638ea8)
    at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:160
#62 0x000055e50e04bc05 in std::__1::__invoke[abi:v15007]<DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}> (__f=...)
    at /usr/local/bin/../include/c++/v1/__functional/invoke.h:394
#63 0x000055e50e04bbdd in