Closed CalvinNeo closed 4 months ago
(gdb) p *(std::__1::mutex*)0x7f0f6aa0b150
$4 = {__m_ = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}}
#27 0x000055af45319eb9 in FastAddPeer::$_29::~$_29 (this=0x7f0f6aa000c8) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/MultiRaft/Disagg/FastAddPeer.cpp:631
631 auto job_func = [server, region_id, new_peer_id, fap_ctx, current_time]() {
(gdb) p region_id
$10 = 1
(gdb) p new_peer_id
$11 = 2333
(gdb) p current_time
$12 = 1713270239304
(gdb) p server
$13 = (DB::EngineStoreServerWrap *) 0x7fffcf9e4348
(gdb) p *(DB::EngineStoreServerWrap *)server
$14 = {tmt = 0x0, proxy_helper = 0x7fffcf9e4380, status = {<std::__1::__atomic_base<DB::EngineStoreServerStatus, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<DB::EngineStoreServerStatus>> = {__a_value = 35}, <No data fields>},
static is_always_lock_free = <optimized out>}, <No data fields>}}
Destroy of
auto fap_ctx = server->tmt->getContext().getSharedContextDisagg()->fap_context;
dead lock in later lambda and ~FastAddPeerContext.
The lambda is the last owner of fap_context. Its quitting results in the destroy of fap_context, thus AsyncTasks.
main: global thread pool join
worker:
When the last task finishes in some worker, count()
could also be zero.
Another
(gdb) bt
#0 0x00007fd1e9ab439a in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1 0x00007fd1e9ab6ba0 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2 0x00007fd1e9eb4d6f in std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) () from /usr/local/lib/x86_64-unknown-linux-gnu/libc++.so.1
#3 0x000055e50e047c81 in std::__1::condition_variable::wait<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::wait()::{lambda()#1}>(std::__1::unique_lock<std::__1::mutex>&, DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::wait()::{lambda()#1}) (
this=0x7fd0f24dd528, __lk=..., __pred=...) at /usr/local/bin/../include/c++/v1/__mutex_base:398
#4 0x000055e50e047b7a in DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::wait (this=0x7fd0f24dd500) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:210
#5 0x000055e50aa84463 in DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::shutdown() (this=0x7fd0f2453df8) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/Utils/AsyncTasks.h:59
#6 0x000055e50aa87393 in DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::~AsyncTasks() (this=0x7fd0f2453df8) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/Utils/AsyncTasks.h:67
#7 0x000055e50aa8734d in std::__1::__destroy_at[abi:v15007]<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, 0>(DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>*) (
__loc=0x7fd0f2453df8) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:63
#8 0x000055e50aa872f5 in std::__1::destroy_at[abi:v15007]<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, 0>(DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>*) (
__loc=0x7fd0f2453df8) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:88
#9 0x000055e50aa872d9 in std::__1::allocator_traits<std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> > >::destroy[abi:v15007]<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, void, void>(std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> >&, DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>*) (__p=0x7fd0f2453df8)
at /usr/local/bin/../include/c++/v1/__memory/allocator_traits.h:317
#10 0x000055e50aa86b0e in std::__1::__shared_ptr_emplace<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> > >::__on_zero_shared() (this=0x7fd0f2453de0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:309
#11 0x000055e5014940b1 in std::__1::__shared_count::__release_shared[abi:v15007]() (this=0x7fd0f2453de0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:174
#12 0x000055e501494059 in std::__1::__shared_weak_count::__release_shared[abi:v15007]() (this=0x7fd0f2453de0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:215
#13 0x000055e50aa8414c in std::__1::shared_ptr<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes> >::~shared_ptr[abi:v15007]() (this=0x7fd0f24dd428) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:702
#14 0x000055e50c4cc69a in DB::FastAddPeerContext::~FastAddPeerContext (this=0x7fd0f24dd418) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/MultiRaft/Disagg/FastAddPeerContext.h:34
#15 0x000055e50c4cc63d in std::__1::__destroy_at[abi:v15007]<DB::FastAddPeerContext, 0>(DB::FastAddPeerContext*) (__loc=0x7fd0f24dd418) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:63
#16 0x000055e50c4cc5e5 in std::__1::destroy_at[abi:v15007]<DB::FastAddPeerContext, 0>(DB::FastAddPeerContext*) (__loc=0x7fd0f24dd418) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:88
--Type <RET> for more, q to quit, c to continue without paging--
#17 0x000055e50c4cc5c9 in std::__1::allocator_traits<std::__1::allocator<DB::FastAddPeerContext> >::destroy[abi:v15007]<DB::FastAddPeerContext, void, void>(std::__1::allocator<DB::FastAddPeerContext>&, DB::FastAddPeerContext*) (__p=0x7fd0f24dd418)
at /usr/local/bin/../include/c++/v1/__memory/allocator_traits.h:317
#18 0x000055e50c4cc46e in std::__1::__shared_ptr_emplace<DB::FastAddPeerContext, std::__1::allocator<DB::FastAddPeerContext> >::__on_zero_shared (this=0x7fd0f24dd400) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:309
#19 0x000055e5014940b1 in std::__1::__shared_count::__release_shared[abi:v15007]() (this=0x7fd0f24dd400) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:174
#20 0x000055e501494059 in std::__1::__shared_weak_count::__release_shared[abi:v15007]() (this=0x7fd0f24dd400) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:215
#21 0x000055e504382e8c in std::__1::shared_ptr<DB::FastAddPeerContext>::~shared_ptr[abi:v15007]() (this=0x7fd0d64000e0) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:702
#22 0x000055e50aa69f39 in FastAddPeer::$_29::~$_29 (this=0x7fd0d64000c8) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/MultiRaft/Disagg/FastAddPeer.cpp:631
#23 0x000055e50aa6c535 in std::__1::__compressed_pair_elem<FastAddPeer::$_29, 0, false>::~__compressed_pair_elem (this=0x7fd0d64000c8) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:30
#24 0x000055e50aa6c655 in std::__1::__compressed_pair<FastAddPeer::$_29, std::__1::allocator<FastAddPeer::$_29> >::~__compressed_pair (this=0x7fd0d64000c8) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:83
#25 0x000055e50aa6cde5 in std::__1::__function::__alloc_func<FastAddPeer::$_29, std::__1::allocator<FastAddPeer::$_29>, DB::FastAddPeerRes ()>::destroy[abi:v15007]() (this=0x7fd0d64000c8) at /usr/local/bin/../include/c++/v1/__functional/function.h:204
#26 0x000055e50aa6c226 in std::__1::__function::__func<FastAddPeer::$_29, std::__1::allocator<FastAddPeer::$_29>, DB::FastAddPeerRes ()>::destroy_deallocate() (this=0x7fd0d64000c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:351
#27 0x000055e504392ad1 in std::__1::__function::__value_func<DB::FastAddPeerRes ()>::~__value_func[abi:v15007]() (this=0x7fd0d6400090) at /usr/local/bin/../include/c++/v1/__functional/function.h:473
#28 0x000055e504383335 in std::__1::function<DB::FastAddPeerRes ()>::~function() (this=0x7fd0d6400090) at /usr/local/bin/../include/c++/v1/__functional/function.h:1184
#29 0x000055e50438d045 in std::__1::__compressed_pair_elem<std::__1::function<DB::FastAddPeerRes ()>, 0, false>::~__compressed_pair_elem() (this=0x7fd0d6400090) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:30
#30 0x000055e50438d065 in std::__1::__compressed_pair<std::__1::function<DB::FastAddPeerRes ()>, std::__1::allocator<std::__1::function<DB::FastAddPeerRes ()> > >::~__compressed_pair() (this=0x7fd0d6400090)
at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:83
#31 0x000055e50438cf66 in std::__1::__packaged_task_func<std::__1::function<DB::FastAddPeerRes ()>, std::__1::allocator<std::__1::function<DB::FastAddPeerRes ()> >, DB::FastAddPeerRes ()>::destroy_deallocate() (this=0x7fd0d6400080)
at /usr/local/bin/../include/c++/v1/future:1686
#32 0x000055e50438caac in std::__1::__packaged_task_function<DB::FastAddPeerRes ()>::~__packaged_task_function() (this=0x7fd0d6407020) at /usr/local/bin/../include/c++/v1/future:1830
#33 0x000055e50438b4c6 in std::__1::packaged_task<DB::FastAddPeerRes ()>::~packaged_task() (this=0x7fd0d6407020) at /usr/local/bin/../include/c++/v1/future:1279
#34 0x000055e50438c78d in std::__1::__destroy_at[abi:v15007]<std::__1::packaged_task<DB::FastAddPeerRes ()>, 0>(std::__1::packaged_task<DB::FastAddPeerRes ()>*) (__loc=0x7fd0d6407020) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:63
#35 0x000055e50438c735 in std::__1::destroy_at[abi:v15007]<std::__1::packaged_task<DB::FastAddPeerRes ()>, 0>(std::__1::packaged_task<DB::FastAddPeerRes ()>*) (__loc=0x7fd0d6407020) at /usr/local/bin/../include/c++/v1/__memory/construct_at.h:88
#36 0x000055e50438c719 in std::__1::allocator_traits<std::__1::allocator<std::__1::packaged_task<DB::FastAddPeerRes ()> > >::destroy[abi:v15007]<std::__1::packaged_task<DB::FastAddPeerRes ()>, void, void>(std::__1::allocator<std::__1::packaged_task<DB::FastAddPeerRes ()--Type <RET> for more, q to quit, c to continue without paging--
> >&, std::__1::packaged_task<DB::FastAddPeerRes ()>*) (__p=0x7fd0d6407020) at /usr/local/bin/../include/c++/v1/__memory/allocator_traits.h:317
#37 0x000055e50438c46e in std::__1::__shared_ptr_emplace<std::__1::packaged_task<DB::FastAddPeerRes ()>, std::__1::allocator<std::__1::packaged_task<DB::FastAddPeerRes ()> > >::__on_zero_shared() (this=0x7fd0d6407000)
at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:309
#38 0x000055e5014940b1 in std::__1::__shared_count::__release_shared[abi:v15007]() (this=0x7fd0d6407000) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:174
#39 0x000055e501494059 in std::__1::__shared_weak_count::__release_shared[abi:v15007]() (this=0x7fd0d6407000) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:215
#40 0x000055e50438b75c in std::__1::shared_ptr<std::__1::packaged_task<DB::FastAddPeerRes ()> >::~shared_ptr[abi:v15007]() (this=0x7fd0d640b190) at /usr/local/bin/../include/c++/v1/__memory/shared_ptr.h:702
#41 0x000055e50438b8ad in DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}::~addTaskWithCancel() (
this=0x7fd0d640b190) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Storages/KVStore/Utils/AsyncTasks.h:271
#42 0x000055e50438eae5 in std::__1::__compressed_pair_elem<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, 0, false>::~__compressed_pair_elem() (this=0x7fd0d640b190) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:30
#43 0x000055e50438ec65 in std::__1::__compressed_pair<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}> >::~__compressed_pair() (
this=0x7fd0d640b190) at /usr/local/bin/../include/c++/v1/__memory/compressed_pair.h:83
#44 0x000055e50438f3f5 in std::__1::__function::__alloc_func<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}>, void ()>::destroy[abi:v15007]() (this=0x7fd0d640b190) at /usr/local/bin/../include/c++/v1/__functional/function.h:204
#45 0x000055e50438e7e6 in std::__1::__function::__func<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}, std::__1::allocator<DB::AsyncTasks<unsigned long, std::__1::function<DB::FastAddPeerRes ()>, DB::FastAddPeerRes>::addTaskWithCancel(unsigned long, std::__1::function<DB::FastAddPeerRes ()>, std::__1::function<void ()>)::{lambda()#1}>, void ()>::destroy_deallocate() (
this=0x7fd0d640b180) at /usr/local/bin/../include/c++/v1/__functional/function.h:351
#46 0x000055e501a33219 in std::__1::__function::__value_func<void ()>::operator=[abi:v15007](decltype(nullptr)) (this=0x7fd1e51fa0c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:503
#47 0x000055e501a331a1 in std::__1::function<void ()>::operator=(decltype(nullptr)) (this=0x7fd1e51fa0c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:1170
#48 0x000055e50e04829a in DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::worker (this=0x7fd0f24dd500, thread_it=...) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:311
#49 0x000055e50e054c64 in DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}::operator()() const (this=0x7fd1e51fa1f0)
--Type <RET> for more, q to quit, c to continue without paging--
at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:160
#50 0x000055e50e054c35 in std::__1::__invoke[abi:v15007]<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&> (__f=...)
at /usr/local/bin/../include/c++/v1/__functional/invoke.h:394
#51 0x000055e50e054c19 in std::__1::__apply_tuple_impl[abi:v15007]<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&, std::__1::__tuple_indices<>) (__f=..., __t=...) at /usr/local/bin/../include/c++/v1/tuple:1789
#52 0x000055e50e054b9d in std::__1::apply[abi:v15007]<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&, std::__1::tuple<>&) (__f=..., __t=...) at /usr/local/bin/../include/c++/v1/tuple:1798
#53 0x000055e50e054b13 in DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}::operator()() (this=0x7fd0d64010c8)
at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.h:236
#54 0x000055e50e054aa5 in std::__1::__invoke[abi:v15007]<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}&> (__f=...)
at /usr/local/bin/../include/c++/v1/__functional/invoke.h:394
#55 0x000055e50e054a65 in std::__1::__invoke_void_return_wrapper<void, true>::__call<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}&>(DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}&) (__args=...) at /usr/local/bin/../include/c++/v1/__functional/invoke.h:479
#56 0x000055e50e054a3d in std::__1::__function::__alloc_func<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}, std::__1::allocator<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}>, void ()>::operator()[abi:v15007]() (this=0x7fd0d64010c8) at /usr/local/bin/../include/c++/v1/__functional/function.h:185
#57 0x000055e50e053e69 in std::__1::__function::__func<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}, std::__1::allocator<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlo--Type <RET> for more, q to quit, c to continue without paging--
balPoolImpl<DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false> >::scheduleImpl<bool>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&)::{lambda()#1}>, void ()>::operator()() (this=0x7fd0d64010c0) at /usr/local/bin/../include/c++/v1/__functional/function.h:359
#58 0x000055e501a2c112 in std::__1::__function::__value_func<void ()>::operator()[abi:v15007]() const (this=0x7fd1e51fa390) at /usr/local/bin/../include/c++/v1/__functional/function.h:512
#59 0x000055e501a1c275 in std::__1::function<void ()>::operator()() const (this=0x7fd1e51fa390) at /usr/local/bin/../include/c++/v1/__functional/function.h:1197
#60 0x000055e50e046358 in DB::ThreadPoolImpl<std::__1::thread>::worker (this=0x7fd1e8a6fa00, thread_it=...) at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:306
#61 0x000055e50e04bc54 in DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}::operator()() const (this=0x7fd1e9638ea8)
at /DATA/disk1/calvin/tiflash/tics4/dbms/src/Common/UniThreadPool.cpp:160
#62 0x000055e50e04bc05 in std::__1::__invoke[abi:v15007]<DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::{lambda()#2}> (__f=...)
at /usr/local/bin/../include/c++/v1/__functional/invoke.h:394
#63 0x000055e50e04bbdd in
Bug Report
Please answer these questions before submitting your issue. Thanks!
1. Minimal reproduce step (Required)
This problem is similar to https://github.com/pingcap/tiflash/pull/8848. We should use explicit shutdown.
2. What did you expect to see? (Required)
3. What did you see instead (Required)
4. What is your TiFlash version? (Required)