search

pingcap / tiflash

The analytical engine for TiDB and TiDB Cloud. Try free: https://tidbcloud.com/free-trial
https://docs.pingcap.com/tidb/stable/tiflash-overview
Apache License 2.0
948 stars 410 forks source link

GRPC core when running tiflash random failpoint tests #5722

Open yibin87 opened 2 years ago

yibin87 commented 2 years ago

Run tiflash random failpoints test, enable "random_join_build_failpoint", tifalsh crashed with little possiblity:


       0xaa1df4d    faultSignalHandler(int, siginfo_t*, void*) [tiflash+178380621]
                    libs/libdaemon/src/BaseDaemon.cpp:221
  0x7f1cadebf630    <unknown symbol> [libpthread.so.0+63024]
  0x7f1cad902387    gsignal [libc.so.6+222087]
  0x7f1cad903a78    __GI_abort [libc.so.6+227960]
  0x7f1cad8fb1a6    __assert_fail_base [libc.so.6+192934]
  0x7f1cad8fb252    __assert_fail [libc.so.6+193106]
       0xc4509a1    grpc_core::RefCount::RefNonZero(grpc_core::DebugLocation const&, char const*) [tiflash+205851041]
                    contrib/grpc/src/core/lib/gprpp/ref_counted.h:137
       0xc4508a5    grpc_stream_ref(grpc_stream_refcount*, char const*) [tiflash+205850789]
                    contrib/grpc/src/core/lib/transport/transport.h:88
       0xc62c401    grpc_chttp2_stream_ref(grpc_chttp2_stream*, char const*) [tiflash+207799297]
                    contrib/grpc/src/core/ext/transport/chttp2/transport/chttp2_transport.cc:606
       0xc62df71    grpc_chttp2_mark_stream_writable(grpc_chttp2_transport*, grpc_chttp2_stream*) [tiflash+207806321]
                    contrib/grpc/src/core/ext/transport/chttp2/transport/chttp2_transport.cc:928
       0xc63210f    read_action_locked(void*, grpc_error*) [tiflash+207823119]
                    contrib/grpc/src/core/ext/transport/chttp2/transport/chttp2_transport.cc:2548
       0xc49c693    grpc_combiner_continue_exec_ctx() [tiflash+206161555]
                    contrib/grpc/src/core/lib/iomgr/combiner.cc:236
       0xc49fad2    grpc_core::ExecCtx::Flush() [tiflash+206174930]
                    contrib/grpc/src/core/lib/iomgr/exec_ctx.cc:156
       0xc4a0778    grpc_core::Executor::RunClosures(char const*, grpc_closure_list) [tiflash+206178168]
                    contrib/grpc/src/core/lib/iomgr/executor.cc:130
       0xc4a0a96    grpc_core::Executor::ThreadMain(void*) [tiflash+206178966]
                    contrib/grpc/src/core/lib/iomgr/executor.cc:248
       0xc7c9906    grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::operator()(void*) const [tiflash+209492230]
                    contrib/grpc/src/core/lib/gprpp/thd_posix.cc:140
       0xc7c9815    grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::__invoke(void*) [tiflash+209491989]
                    contrib/grpc/src/core/lib/gprpp/thd_posix.cc:110
  0x7f1cadeb7ea5    start_thread [libpthread.so.0+32421]"] [thread_id=1037]

another core stack:
[2022/08/31 14:58:20.695 +08:00] [ERROR] [BaseDaemon.cpp:570] ["BaseDaemon:
       0xaa891ddfaultSignalHandler(int, siginfo_t*, void*) [tiflash+178819549]
                libs/libdaemon/src/BaseDaemon.cpp:221
  0x7fd07bf9a630<unknown symbol> [libpthread.so.0+63024]
  0x7fd07b9dd387gsignal [libc.so.6+222087]
  0x7fd07b9dea78__GI_abort [libc.so.6+227960]
  0x7fd07b9d61a6__assert_fail_base [libc.so.6+192934]
  0x7fd07b9d6252__assert_fail [libc.so.6+193106]
       0xc4b92e1grpc_core::RefCount::RefNonZero(grpc_core::DebugLocation const&, char const*) [tiflash+206279393]
                contrib/grpc/src/core/lib/gprpp/ref_counted.h:137
       0xc4b91e5grpc_stream_ref(grpc_stream_refcount*, char const*) [tiflash+206279141]
                contrib/grpc/src/core/lib/transport/transport.h:88
       0xc694d41grpc_chttp2_stream_ref(grpc_chttp2_stream*, char const*) [tiflash+208227649]
                contrib/grpc/src/core/ext/transport/chttp2/transport/chttp2_transport.cc:606
       0xc6968b1grpc_chttp2_mark_stream_writable(grpc_chttp2_transport*, grpc_chttp2_stream*) [tiflash+208234673]
                contrib/grpc/src/core/ext/transport/chttp2/transport/chttp2_transport.cc:928
       0xc69aa4fread_action_locked(void*, grpc_error*) [tiflash+208251471]
                contrib/grpc/src/core/ext/transport/chttp2/transport/chttp2_transport.cc:2548
       0xc504fd3grpc_combiner_continue_exec_ctx() [tiflash+206589907]
                contrib/grpc/src/core/lib/iomgr/combiner.cc:236
       0xc508412grpc_core::ExecCtx::Flush() [tiflash+206603282]
                contrib/grpc/src/core/lib/iomgr/exec_ctx.cc:156
       0xc50ca9cpollset_work(grpc_pollset*, grpc_pollset_worker**, long) [tiflash+206621340]
                contrib/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:1136
       0xc50b393pollset_work(grpc_pollset*, grpc_pollset_worker**, long) [tiflash+206615443]
                contrib/grpc/src/core/lib/iomgr/ev_posix.cc:324
       0xc517b8egrpc_pollset_work(grpc_pollset*, grpc_pollset_worker**, long) [tiflash+206666638]
                contrib/grpc/src/core/lib/iomgr/pollset.cc:48
       0xc538ffbcq_next(grpc_completion_queue*, gpr_timespec, void*) [tiflash+206802939]
                contrib/grpc/src/core/lib/surface/completion_queue.cc:1038
       0xc53854agrpc_completion_queue_next [tiflash+206800202]
                contrib/grpc/src/core/lib/surface/completion_queue.cc:1114
       0xc4783acgrpc_impl::CompletionQueue::AsyncNextInternal(void**, bool*, gpr_timespec) [tiflash+206013356]
                contrib/grpc/src/cpp/common/completion_queue_cc.cc:48
       0x4737035grpc_impl::CompletionQueue::Next(void**, bool*) [tiflash+74674229]
                contrib/grpc/include/grpcpp/impl/codegen/completion_queue_impl.h:176
       0x47367e8DB::(anonymous namespace)::handleRpcs(grpc_impl::ServerCompletionQueue*, std::__1::shared_ptr<DB::Logger> const&) [tiflash+74672104]
                dbms/src/Server/FlashGrpcServerHolder.cpp:42
       0x473917fDB::FlashGrpcServerHolder::FlashGrpcServerHolder(DB::Context&, Poco::Util::LayeredConfiguration&, DB::TiFlashSecurityConfig&, DB::TiFlashRaftConfig const&, std::__1::shared_ptr<DB::Logger> const&)::$_6::operator()() const [tiflash+74682751]
                dbms/src/Server/FlashGrpcServerHolder.cpp:156```
SeaRise commented 1 year ago 
2023-06-06 10:44:23 
{"log":"[BaseDaemon.cpp:569] [\"\
       0xacf0e5d\\tfaultSignalHandler(int, siginfo_t*, void*) [tiflash+181341789]\
                \\tlibs/libdaemon/src/BaseDaemon.cpp:220\
  0x7f592cca8d90\\t<unknown symbol> [libc.so.6+347536]\
       0xc9e85af\\texec_ctx_run(grpc_closure*) [tiflash+211715503]\
                \\tcontrib/grpc/src/core/lib/iomgr/exec_ctx.cc:49\
       0xc9e8461\\tgrpc_core::ExecCtx::Flush() [tiflash+211715169]\
                \\tcontrib/grpc/src/core/lib/iomgr/exec_ctx.cc:164\
       0xca1a0e2\\trun_some_timers() [tiflash+211919074]\
                \\tcontrib/grpc/src/core/lib/iomgr/timer_manager.cc:134\
       0xca19ea7\\ttimer_main_loop() [tiflash+211918503]\
                \\tcontrib/grpc/src/core/lib/iomgr/timer_manager.cc:237\
       0xca19e1f\\ttimer_thread(void*) [tiflash+211918367]\
                \\tcontrib/grpc/src/core/lib/iomgr/timer_manager.cc:284\
       0xcdf2106\\tgrpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::operator()(void*) const [tiflash+215949574]\
                \\tcontrib/grpc/src/core/lib/gprpp/thd_posix.cc:140\
       0xcdf2015\\tgrpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::__invoke(void*) [tiflash+215949333]\
                \\tcontrib/grpc/src/core/lib/gprpp/thd_posix.cc:110\
  0x7f592ccf3802\\tstart_thread [libc.so.6+653314]\"] [source=BaseDaemon] [thread_id=1039]","level":"ERROR","container":"data0","pod":"tc-tiflash-2","namespace":"endless-htap-pipeline-random-failpoint-tps-1804838-1-6"}
2023-06-06 10:44:23 
{"level":"ERROR","container":"data0","pod":"tc-tiflash-2","log":"[BaseDaemon.cpp:569] [\"\
       0xacf0e5d\\tfaultSignalHandler(int, siginfo_t*, void*) [tiflash+181341789]\
                \\tlibs/libdaemon/src/BaseDaemon.cpp:220\
  0x7f592cca8d90\\t<unknown symbol> [libc.so.6+347536]\
       0xc9e85af\\texec_ctx_run(grpc_closure*) [tiflash+211715503]\
                \\tcontrib/grpc/src/core/lib/iomgr/exec_ctx.cc:49\
       0xc9e8461\\tgrpc_core::ExecCtx::Flush() [tiflash+211715169]\
                \\tcontrib/grpc/src/core/lib/iomgr/exec_ctx.cc:164\
       0xca1a0e2\\trun_some_timers() [tiflash+211919074]\
                \\tcontrib/grpc/src/core/lib/iomgr/timer_manager.cc:134\
       0xca19ea7\\ttimer_main_loop() [tiflash+211918503]\
                \\tcontrib/grpc/src/core/lib/iomgr/timer_manager.cc:237\
       0xca19e1f\\ttimer_thread(void*) [tiflash+211918367]\
                \\tcontrib/grpc/src/core/lib/iomgr/timer_manager.cc:284\
       0xcdf2106\\tgrpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::operator()(void*) const [tiflash+215949574]\
                \\tcontrib/grpc/src/core/lib/gprpp/thd_posix.cc:140\
       0xcdf2015\\tgrpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::__invoke(void*) [tiflash+215949333]\
                \\tcontrib/grpc/src/core/lib/gprpp/thd_posix.cc:110\
  0x7f592ccf3802\\tstart_thread [libc.so.6+653314]\"] [source=BaseDaemon] [thread_id=1039]","namespace":"endless-htap-pipeline-random-failpoint-tps-1804838-1-6"}

Also found in pipeline random failpoint, but with different logs.

yibin87 commented 7 months ago

New grpc core stack:

[ERROR] [BaseDaemon.cpp:570] [\"\
       0x61b6fe1\faultSignalHandler(int, siginfo_t*, void*) [tiflash+102461409]\
                \libs/libdaemon/src/BaseDaemon.cpp:221\
  0x7f16eee37630\<unknown symbol> [libpthread.so.0+63024]\
  0x7f16ee16c387\gsignal [libc.so.6+222087]\
  0x7f16ee16da78\__GI_abort [libc.so.6+227960]\
       0x76cfbcf\call_start_batch(grpc_call*, grpc_op const*, unsigned long, void*, int) [tiflash+124582863]\
                \contrib/grpc/src/core/lib/surface/call.cc:0\
       0x76ce7ef\grpc_call_start_batch [tiflash+124577775]\
                \contrib/grpc/src/core/lib/surface/call.cc:1761\
       0x6d74dff\grpc::internal::CallOpSet<grpc::internal::CallOpSendInitialMetadata, grpc::internal::CallOpSendMessage, grpc::internal::CallOpServerSendStatus, grpc::internal::CallNoOp<4>, grpc::internal::CallNoOp<5>, grpc::internal::CallNoOp<6> >::ContinueFillOpsAfterInterception() [tiflash+114773503]\
                \contrib/grpc/include/grpcpp/impl/codegen/call_op_set.h:971\
       0x6d724b6\grpc::ServerAsyncWriter<mpp::MPPDataPacket>::Write(mpp::MPPDataPacket const&, void*) [tiflash+114762934]\
                \contrib/grpc/include/grpcpp/impl/codegen/async_stream.h:876\
       0x6d70ae4\DB::EstablishCallData::tryConnectTunnel() [tiflash+114756324]\
                \dbms/src/Flash/EstablishCall.cpp:163\
       0x17fef95\DB::(anonymous namespace)::handleRpcs(grpc::ServerCompletionQueue*, std::__1::shared_ptr<DB::Logger> const&) [tiflash+25161621]\
                \dbms/src/Server/FlashGrpcServerHolder.cpp:55\
       0x17feaed\void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, std::__1::thread DB::ThreadFactory::newThread<DB::FlashGrpcServerHolder::FlashGrpcServerHolder(DB::Context&, Poco::Util::LayeredConfiguration&, DB::TiFlashSecurityConfig&, DB::TiFlashRaftConfig const&, std::__1::shared_ptr<DB::Logger> const&)::$_5>(bool, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, DB::FlashGrpcServerHolder::FlashGrpcServerHolder(DB::Context&, Poco::Util::LayeredConfiguration&, DB::TiFlashSecurityConfig&, DB::TiFlashRaftConfig const&, std::__1::shared_ptr<DB::Logger> const&)::$_5&&)::'lambda'(auto&&...)> >(void*) [tiflash+25160429]\
                \/usr/local/bin/../include/c++/v1/thread:291\
yibin87 commented 3 months ago

New met grpc core crash: 

[ERROR] [BaseDaemon.cpp:569] [\"
       0x756bfb1\\tfaultSignalHandler(int, siginfo_t*, void*) [tiflash+123125681]
                \\tlibs/libdaemon/src/BaseDaemon.cpp:220
  0x7fa5e1522db0\\t<unknown symbol> [libc.so.6+347568]
  0x7fa5e156f54c\\t__pthread_kill_implementation [libc.so.6+660812]
  0x7fa5e1522d06\\t__GI_raise [libc.so.6+347398]
  0x7fa5e14f67f3\\tabort [libc.so.6+165875]
       0x8b37b6f\\tcall_start_batch(grpc_call*, grpc_op const*, unsigned long, void*, int) [tiflash+145980271]
                \\tcontrib/grpc/src/core/lib/surface/call.cc:0
       0x8b3678f\\tgrpc_call_start_batch [tiflash+145975183]
                \\tcontrib/grpc/src/core/lib/surface/call.cc:1761
       0x8124f0f\\tgrpc::internal::CallOpSet<grpc::internal::CallOpSendInitialMetadata, grpc::internal::CallOpSendMessage, grpc::internal::CallOpServerSendStatus, grpc::internal::CallNoOp<4>, grpc::internal::CallNoOp<5>, grpc::internal::CallNoOp<6> >::ContinueFillOpsAfterInterception() [tiflash+135417615]
                \\tcontrib/grpc/include/grpcpp/impl/codegen/call_op_set.h:971 
       0x8123656\\tgrpc::ServerAsyncWriter<mpp::MPPDataPacket>::Write(mpp::MPPDataPacket const&, void*) [tiflash+135411286]
                \\tcontrib/grpc/include/grpcpp/impl/codegen/async_stream.h:876
       0x81211ae\\tDB::EstablishCallData::tryConnectTunnel() [tiflash+135401902]
                \\tdbms/src/Flash/EstablishCall.cpp:166
       0x1e91395\\tDB::(anonymous namespace)::handleRpcs(grpc::ServerCompletionQueue*, std::__1::shared_ptr<DB::Logger> const&) [tiflash+32052117]
                \\tdbms/src/Server/FlashGrpcServerHolder.cpp:63
       0x1e90eed\\tvoid* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, std::__1::thread DB::ThreadFactory::newThread<DB::FlashGrpcServerHolder::FlashGrpcServerHolder(DB::Context&, Poco::Util::LayeredConfiguration&, DB::TiFlashRaftConfig const&, std::__1::shared_ptr<DB::Logger> const&)::$_5>(bool, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, DB::FlashGrpcServerHolder::FlashGrpcServerHolder(DB::Context&, Poco::Util::LayeredConfiguration&, DB::TiFlashRaftConfig const&, std::__1::shared_ptr<DB::Logger> const&)::$_5&&)::'lambda'(auto&&...)> >(void*) [tiflash+32050925]
                \\t/usr/local/bin/../include/c++/v1/thread:291
  0x7fa5e156d802\\tstart_thread [libc.so.6+653314]\"] [source=BaseDaemon] [thread_id=896]

[ERROR] [BaseDaemon.cpp:569] [\"
       0x756bfb1\\tfaultSignalHandler(int, siginfo_t*, void*) [tiflash+123125681]
                \\tlibs/libdaemon/src/BaseDaemon.cpp:220
  0x7fa5e1522db0\\t<unknown symbol> [libc.so.6+347568]
  0x7fa5e156f54c\\t__pthread_kill_implementation [libc.so.6+660812]    
  0x7fa5e1522d06\\t__GI_raise [libc.so.6+347398]
  0x7fa5e14f67f3\\tabort [libc.so.6+165875]
       0x8b37b6f\\tcall_start_batch(grpc_call*, grpc_op const*, unsigned long, void*, int) [tiflash+145980271]
                \\tcontrib/grpc/src/core/lib/surface/call.cc:0
       0x8b3678f\\tgrpc_call_start_batch [tiflash+145975183]
                \\tcontrib/grpc/src/core/lib/surface/call.cc:1761
       0x8124f0f\\tgrpc::internal::CallOpSet<grpc::internal::CallOpSendInitialMetadata, grpc::internal::CallOpSendMessage, grpc::internal::CallOpServerSendStatus, grpc::internal::CallNoOp<4>, grpc::internal::CallNoOp<5>, grpc::internal::CallNoOp<6> >::ContinueFillOpsAfterInterception() [tiflash+135417615]
                \\tcontrib/grpc/include/grpcpp/impl/codegen/call_op_set.h:971
       0x8123656\\tgrpc::ServerAsyncWriter<mpp::MPPDataPacket>::Write(mpp::MPPDataPacket const&, void*) [tiflash+135411286]
                \\tcontrib/grpc/include/grpcpp/impl/codegen/async_stream.h:876
       0x81211ae\\tDB::EstablishCallData::tryConnectTunnel() [tiflash+135401902]
                \\tdbms/src/Flash/EstablishCall.cpp:166
       0x1e91395\\tDB::(anonymous namespace)::handleRpcs(grpc::ServerCompletionQueue*, std::__1::shared_ptr<DB::Logger> const&) [tiflash+32052117]
                \\tdbms/src/Server/FlashGrpcServerHolder.cpp:63
       0x1e90eed\\tvoid* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, std::__1::thread DB::ThreadFactory::newThread<DB::FlashGrpcServerHolder::FlashGrpcServerHolder(DB::Context&, Poco::Util::LayeredConfiguration&, DB::TiFlashRaftConfig const&, std::__1::shared_ptr<DB::Logger> const&)::$_5>(bool, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, DB::FlashGrpcServerHolder::FlashGrpcServerHolder(DB::Context&, Poco::Util::LayeredConfiguration&, DB::TiFlashRaftConfig const&, std::__1::shared_ptr<DB::Logger> const&)::$_5&&)::'lambda'(auto&&...)> >(void*) [tiflash+32050925]
                \\t/usr/local/bin/../include/c++/v1/thread:291
  0x7fa5e156d802\\tstart_thread [libc.so.6+653314]\"] [source=BaseDaemon] [thread_id=896]
lilinghai commented 2 months ago

another

[2024/09/02 02:52:56.091 +00:00] [ERROR] [BaseDaemon.cpp:376] [########################################] [source=BaseDaemon] [thread_id=917]
[2024/09/02 02:52:56.092 +00:00] [ERROR] [BaseDaemon.cpp:377] ["(from thread 829) Received signal Aborted(6)."] [source=BaseDaemon] [thread_id=917]
[2024/09/02 02:52:56.112 +00:00] [ERROR] [BaseDaemon.cpp:569] ["\n       0x756bfb1\tfaultSignalHandler(int, siginfo_t*, void*) [tiflash+123125681]\n                \tlibs/libdaemon/src/BaseDaemon.cpp:220\n  0x7f7952618db0\t<unknown symbol> [libc.so.6+347568]\n  0x7f795266554c\t__pthread_kill_implementation [libc.so.6+660812]\n  0x7f7952618d06\t__GI_raise [libc.so.6+347398]\n  0x7f79525ec7f3\tabort [libc.so.6+165875]\n       0x8b37b6f\tcall_start_batch(grpc_call*, grpc_op const*, unsigned long, void*, int) [tiflash+145980271]\n                \tcontrib/grpc/src/core/lib/surface/[call.cc:0](http://call.cc:0/)\n       0x8b3678f\tgrpc_call_start_batch [tiflash+145975183]\n                \tcontrib/grpc/src/core/lib/surface/[call.cc:1761](http://call.cc:1761/)\n       0x8124f0f\tgrpc::internal::CallOpSet<grpc::internal::CallOpSendInitialMetadata, grpc::internal::CallOpSendMessage, grpc::internal::CallOpServerSendStatus, grpc::internal::CallNoOp<4>, grpc::internal::CallNoOp<5>, grpc::internal::CallNoOp<6> >::ContinueFillOpsAfterInterception() [tiflash+135417615]\n                \tcontrib/grpc/include/grpcpp/impl/codegen/call_op_set.h:971\n       0x8123656\tgrpc::ServerAsyncWriter<mpp::MPPDataPacket>::Write(mpp::MPPDataPacket const&, void*) [tiflash+135411286]\n                \tcontrib/grpc/include/grpcpp/impl/codegen/async_stream.h:876\n       0x81211ae\tDB::EstablishCallData::tryConnectTunnel() [tiflash+135401902]\n                \tdbms/src/Flash/EstablishCall.cpp:166\n       0x1e91395\tDB::(anonymous namespace)::handleRpcs(grpc::ServerCompletionQueue*, std::__1::shared_ptr<DB::Logger> const&) [tiflash+32052117]\n                \tdbms/src/Server/FlashGrpcServerHolder.cpp:63\n       0x1e90eed\tvoid* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, std::__1::thread DB::ThreadFactory::newThread<DB::FlashGrpcServerHolder::FlashGrpcServerHolder(DB::Context&, Poco::Util::LayeredConfiguration&, DB::TiFlashRaftConfig const&, std::__1::shared_ptr<DB::Logger> const&)::$_5>(bool, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, DB::FlashGrpcServerHolder::FlashGrpcServerHolder(DB::Context&, Poco::Util::LayeredConfiguration&, DB::TiFlashRaftConfig const&, std::__1::shared_ptr<DB::Logger> const&)::$_5&&)::'lambda'(auto&&...)> >(void*) [tiflash+32050925]\n                \t/usr/local/bin/../include/c++/v1/thread:291\n  0x7f7952663802\tstart_thread [libc.so.6+653314]"] [source=BaseDaemon] [thread_id=917]