search

yugabyte / yugabyte-db

YugabyteDB - the cloud native distributed SQL database for mission-critical applications.
https://www.yugabyte.com
Other
8.99k stars 1.07k forks source link

Check failed: task_state == MonitoredTaskState::kWaiting State: kScheduling #5447

Open amitanandaiyer opened 4 years ago

amitanandaiyer commented 4 years ago

Jira Link: DB-10812 failure while running ybd tsan --cxx-test integration-tests_cassandra_cpp_driver-test --gtest_filter CppCassandraDriverTest.TestDeleteAndCreateIndex -n 100 --tp 1 with D9182

[m-1] Fatal failure details written to /net/dev-server-amitanand2/share/logs/repeat_unit_test/cassandra_cpp_driver-test/CppCassandraDriverTest.TestDeleteAndCreateIndex/2020-08-19T23_23_      15/91.fatal_failure_details.m-1.2020-08-20T03_47_15.pid29858.txt
59849 [m-1] F20200820 03:47:15 ../../src/yb/master/async_rpc_tasks.cc:134] Check failed: task_state == MonitoredTaskState::kWaiting State: kScheduling
59850 [m-1]     @     0x7ff982620a08  yb::LogFatalHandlerSink::send(int, char const*, char const*, int, tm const*, char const*, unsigned long) (src/yb/util/logging.cc:474)
59851 [m-1]     @     0x7ff981a360dd
59852 [m-1]     @     0x7ff981a2ce12
59853 [m-1]     @     0x7ff981a2de5c
59854 [m-1]     @     0x7ff981a32afa
59855 [m-1]     @     0x7ff98deec633  yb::master::RetryingTSRpcTask::Run() (src/yb/master/async_rpc_tasks.cc:0)
59856 [m-1]     @     0x7ff98deedb8f  yb::master::RetryingTSRpcTask::RunDelayedTask(yb::Status const&) (src/yb/master/async_rpc_tasks.cc:342)
59857 [m-1]     @     0x7ff98defeb75  decltype(*(std::forward<shared_ptr<yb::master::RetryingTSRpcTask>&>(fp0)).*fp(std::forward<yb::Status const&>(fp1))) std::__invoke<void (yb::master::Retr      yingTSRpcTask::*&)(yb::Status const&), shared_ptr<yb::master::RetryingTSRpcTask>&, yb::Status const&, void>(void (yb::master::RetryingTSRpcTask::*&)(yb::Status const&), shared_ptr<yb::m      aster::RetryingTSRpcTask>&, yb::Status const&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/installed/tsan/libcxx/include/c++/v1/type_traits:4286)
59858 [m-1]     @     0x7ff98defeb75  std::__bind_return<void (yb::master::RetryingTSRpcTask::*)(yb::Status const&), tuple<shared_ptr<yb::master::RetryingTSRpcTask>, std::placeholders::__ph<1      > >, tuple<yb::Status const&>, __is_valid_bind_return<void (yb::master::RetryingTSRpcTask::*)(yb::Status const&), tuple<shared_ptr<yb::master::RetryingTSRpcTask>, std::placeholders::__p      h<1> >, tuple<yb::Status const&> >::value>::type std::__apply_functor<void (yb::master::RetryingTSRpcTask::*)(yb::Status const&), tuple<shared_ptr<yb::master::RetryingTSRpcTask>, std::p      laceholders::__ph<1> >, 0ul, 1ul, tuple<yb::Status const&> >(void (yb::master::RetryingTSRpcTask::*&)(yb::Status const&), tuple<shared_ptr<yb::master::RetryingTSRpcTask>, std::placehold      ers::__ph<1> >&, std::__tuple_indices<0ul, 1ul>, tuple<yb::Status const&>&&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/installed/tsan/libcxx/inc      lude/c++/v1/functional:2216)
59859 [m-1]     @     0x7ff98defeb75  std::__bind_return<void (yb::master::RetryingTSRpcTask::*)(yb::Status const&), tuple<shared_ptr<yb::master::RetryingTSRpcTask>, std::placeholders::__ph<1      > >, tuple<yb::Status const&>, __is_valid_bind_return<void (yb::master::RetryingTSRpcTask::*)(yb::Status const&), tuple<shared_ptr<yb::master::RetryingTSRpcTask>, std::placeholders::__p      h<1> >, tuple<yb::Status const&> >::value>::type std::__bind<void (yb::master::RetryingTSRpcTask::*)(yb::Status const&), shared_ptr<yb::master::RetryingTSRpcTask>, std::placeholders::__      ph<1> const&>::operator()<yb::Status const&>(yb::Status const&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/installed/tsan/libcxx/include/c++/v1/f      unctional:2249)
59860 [m-1]     @     0x7ff98defeb75  boost::detail::function::void_function_obj_invoker1<std::__bind<void (yb::master::RetryingTSRpcTask::*)(yb::Status const&), shared_ptr<yb::master::Retryi      ngTSRpcTask>, std::placeholders::__ph<1> const&>, void, yb::Status const&>::invoke(boost::detail::function::function_buffer&, yb::Status const&) (/opt/yb-build/thirdparty/yugabyte-db-th      irdparty-v20200707012818-49ca690c1f-centos/installed/tsan/include/boost/function/function_template.hpp:158)
59861 [m-1]     @     0x7ff989fb9749  boost::function1<void, yb::Status const&>::operator()(yb::Status const&) const (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1      f-centos/installed/tsan/include/boost/function/function_template.hpp:763)
59862 [m-1]     @     0x7ff98408861a  yb::rpc::DelayedTask::TimerHandler(ev::timer&, int) (src/yb/rpc/reactor.cc:855)
59863 [m-1]     @     0x7ff9840949fa  void ev::base<ev_timer, ev::timer>::method_thunk<yb::rpc::DelayedTask, &(yb::rpc::DelayedTask::TimerHandler(ev::timer&, int))>(ev_loop*, ev_timer*, int)       (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/installed/common/include/ev++.h:479)
59864 [m-1]     @     0x7ff9813dbb6a
59865 [m-1]     @     0x7ff9813dfc79
59866 [m-1]     @     0x7ff98401ab68  ev::loop_ref::run(int) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/installed/common/include/ev++.h:211)
59867 [m-1]     @     0x7ff984081f6c  yb::rpc::Reactor::RunThread() (src/yb/rpc/reactor.cc:482)
59868 [m-1]     @     0x7ff98408e3e3  decltype(*(std::forward<yb::rpc::Reactor*&>(fp0)).*fp()) std::__invoke<void (yb::rpc::Reactor::*&)(), yb::rpc::Reactor*&, void>(void (yb::rpc::Reactor::*      &)(), yb::rpc::Reactor*&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/installed/tsan/libcxx/include/c++/v1/type_traits:4286)
59869 [m-1]     @     0x7ff98408e3e3  std::__bind_return<void (yb::rpc::Reactor::*)(), tuple<yb::rpc::Reactor*>, tuple<>, __is_valid_bind_return<void (yb::rpc::Reactor::*)(), tuple<yb::rpc::R      eactor*>, tuple<> >::value>::type std::__apply_functor<void (yb::rpc::Reactor::*)(), tuple<yb::rpc::Reactor*>, 0ul, tuple<> >(void (yb::rpc::Reactor::*&)(), tuple<yb::rpc::Reactor*>&, s      td::__tuple_indices<0ul>, tuple<>&&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/installed/tsan/libcxx/include/c++/v1/functional:2216)
59870 [m-1]     @     0x7ff98408e3e3  std::__bind_return<void (yb::rpc::Reactor::*)(), tuple<yb::rpc::Reactor*>, tuple<>, __is_valid_bind_return<void (yb::rpc::Reactor::*)(), tuple<yb::rpc::R      eactor*>, tuple<> >::value>::type std::__bind<void (yb::rpc::Reactor::* const&)(), yb::rpc::Reactor* const&>::operator()<>() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v2020070701      2818-49ca690c1f-centos/installed/tsan/libcxx/include/c++/v1/functional:2249)
59871 [m-1]     @     0x7ff98408e3e3  decltype(std::forward<std::__bind<void (yb::rpc::Reactor::* const&)(), yb::rpc::Reactor* const&>&>(fp)()) std::__invoke<std::__bind<void (yb::rpc::Reacto      r::* const&)(), yb::rpc::Reactor* const&>&>(std::__bind<void (yb::rpc::Reactor::* const&)(), yb::rpc::Reactor* const&>&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818      -49ca690c1f-centos/installed/tsan/libcxx/include/c++/v1/type_traits:4345)
59872 [m-1]     @     0x7ff98408e3e3  void std::__invoke_void_return_wrapper<void>::__call<std::__bind<void (yb::rpc::Reactor::* const&)(), yb::rpc::Reactor* const&>&>(std::__bind<void (yb::r      pc::Reactor::* const&)(), yb::rpc::Reactor* const&>&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/installed/tsan/libcxx/include/c++/v1/__functiona      l_base:349)
59873 [m-1]     @     0x7ff98408e2ed  std::__function::__func<std::__bind<void (yb::rpc::Reactor::* const&)(), yb::rpc::Reactor* const&>, std::allocator<std::__bind<void (yb::rpc::Reactor::*       const&)(), yb::rpc::Reactor* const&> >, void ()>::operator()() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/installed/tsan/libcxx/include/c++/v1/fu      nctional:1562)
59874 [m-1]     @     0x7ff98d70a934  std::function<void ()>::operator()() const (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/installed/tsan/libcxx/inclu      de/c++/v1/functional:1913)
59875 [m-1]     @     0x7ff9826e3ae9  yb::Thread::SuperviseThread(void*) (src/yb/util/thread.cc:760)
59876 [m-1]     @           0x456bad  __tsan_thread_start_func (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20200707012818-49ca690c1f-centos/src/llvm-7.1.0.src/projects/compiler-rt/lib/t      san/rtl/tsan_interceptors.cc:955)
59877 [m-1]     @     0x7ff97e0bd693  start_thread (/tmp/glibc-20181130-26094-cs1x60/glibc-2.23/nptl/pthread_create.c:333)
59878 [m-1]     @     0x7ff97dbe841c  (unknown) (sysdeps/unix/sysv/linux/x86_64/clone.S:109)
59879 [m-1]     @ 0xffffffffffffffff
59880 [m-1]
59881 [m-1] *** Check failure stack trace: ***
amitanandaiyer commented 4 years ago

may be a known issue? @bmatican 

  // TODO(bogdan): There is a race between scheduling and running and can cause this to fail.
  // Should look into removing the kScheduling state, if not needed, and simplifying the state
  // transitions!
  DCHECK(task_state == MonitoredTaskState::kWaiting) << "State: " << ToString(task_state);
amitanandaiyer commented 4 years ago

91.log.gz

rthallamko3 commented 7 months ago

@amitanandaiyer , Do you know if this is still relevant? If not, can we close it?