search

yugabyte / yugabyte-db

YugabyteDB - the cloud native distributed SQL database for mission-critical applications.
https://www.yugabyte.com
Other
9.04k stars 1.08k forks source link

DCHECK in HdrHistogram increment during initdb #9229

Open mbautin opened 3 years ago

mbautin commented 3 years ago

Jira Link: DB-10502 Failed here: https://github.com/yugabyte/yugabyte-db/runs/3024850292 Commit: 362723816c20eb4650861729d4e53e8bf42c3174

Log: https://gist.githubusercontent.com/mbautin/8f5983db88351330bef1d0782dc24be2/raw

2021-07-09T03:28:15.7051894Z [m-1] Fatal failure details written to /opt/yb-build/yugabyte-db/build/debug-clang11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.fatal_failure_details.m-1.2021-07-09T03_28_12.pid193474.txt
2021-07-09T03:28:15.7054756Z [m-1] F20210709 03:28:12 ../../src/yb/util/hdr_histogram.cc:180] Check failed: value >= 0 (-1 vs. 0) 
2021-07-09T03:28:15.7056540Z [m-1]     @     0x7f42563470a0  yb::LogFatalHandlerSink::send(int, char const*, char const*, int, tm const*, char const*, unsigned long) (src/yb/util/logging.cc:474)
2021-07-09T03:28:15.7057604Z [m-1]     @     0x7f42559ab84b 
2021-07-09T03:28:15.7058188Z [m-1]     @     0x7f42559a548e 
2021-07-09T03:28:15.7058753Z [m-1]     @     0x7f42559a5da7 
2021-07-09T03:28:15.7059328Z [m-1]     @     0x7f42559a8f5e 
2021-07-09T03:28:15.7060491Z [m-1]     @     0x7f4256338def  yb::HdrHistogram::IncrementBy(long, long) (src/yb/util/hdr_histogram.cc:0)
2021-07-09T03:28:15.7061902Z [m-1]     @     0x7f4256338c2d  yb::HdrHistogram::Increment(long) (src/yb/util/hdr_histogram.cc:176)
2021-07-09T03:28:15.7063176Z [m-1]     @     0x7f425636602c  yb::Histogram::Increment(long) (src/yb/util/metrics.cc:885)
2021-07-09T03:28:15.7064335Z [m-1]     @     0x7f4258903abc  ~ReadRpc (src/yb/client/async_rpc.cc:705)
2021-07-09T03:28:15.7067478Z [m-1]     @     0x7f4258925bc5  std::__shared_ptr_emplace<yb::client::internal::ReadRpc, std::allocator<yb::client::internal::ReadRpc> >::__on_zero_shared() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/memory:3503)
2021-07-09T03:28:15.7071590Z [m-1]     @     0x7f42588fdb94  std::__shared_count::__release_shared() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/memory:3341)
2021-07-09T03:28:15.7075314Z [m-1]     @     0x7f42588fdb4d  std::__shared_weak_count::__release_shared() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/memory:3383)
2021-07-09T03:28:15.7078694Z [m-1]     @     0x7f4258904e01  ~shared_ptr (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/memory:4098)
2021-07-09T03:28:15.7082326Z [m-1]     @     0x7f425890530c  shared_ptr<yb::rpc::RpcCommand>::reset() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/memory:4233)
2021-07-09T03:28:15.7084858Z [m-1]     @     0x7f4258900fcb  yb::client::internal::AsyncRpc::Finished(yb::Status const&) (src/yb/client/async_rpc.cc:212)
2021-07-09T03:28:15.7088757Z [m-1]     @     0x7f425890e5a4  decltype(*(std::forward<yb::client::internal::ReadRpc*&>(fp0)).*fp(std::forward<yb::Status::OK&>(fp1))) std::__invoke<void (yb::client::internal::AsyncRpc::*&)(yb::Status const&), yb::client::internal::ReadRpc*&, yb::Status::OK&, void>(void (yb::client::internal::AsyncRpc::*&)(yb::Status const&), yb::client::internal::ReadRpc*&, yb::Status::OK&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/type_traits:3840)
2021-07-09T03:28:15.7096159Z [m-1]     @     0x7f425890e53a  std::__bind_return<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>, tuple<>, __is_valid_bind_return<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>, tuple<> >::value>::type std::__apply_functor<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>, 0ul, 1ul, tuple<> >(void (yb::client::internal::AsyncRpc::*&)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>&, std::__tuple_indices<0ul, 1ul>, tuple<>&&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2853)
2021-07-09T03:28:15.7103414Z [m-1]     @     0x7f425890e4e4  std::__bind_return<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>, tuple<>, __is_valid_bind_return<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>, tuple<> >::value>::type std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>::operator()<>() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2886)
2021-07-09T03:28:15.7109805Z [m-1]     @     0x7f425890e4b0  decltype(std::forward<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>&>(fp)()) std::__invoke<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>&>(std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/type_traits:3899)
2021-07-09T03:28:15.7115870Z [m-1]     @     0x7f425890e480  void std::__invoke_void_return_wrapper<void>::__call<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>&>(std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/__functional_base:348)
2021-07-09T03:28:15.7121417Z [m-1]     @     0x7f425890e460  std::__function::__alloc_func<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>, std::allocator<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK> >, void ()>::operator()() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1557)
2021-07-09T03:28:15.7126951Z [m-1]     @     0x7f425890da4c  std::__function::__func<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>, std::allocator<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK> >, void ()>::operator()() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1731)
2021-07-09T03:28:15.7131568Z [m-1]     @     0x7f425891a722  std::__function::__value_func<void ()>::operator()() const (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1884)
2021-07-09T03:28:15.7135498Z [m-1]     @     0x7f425891a708  std::function<void ()>::operator()() const (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2556)
2021-07-09T03:28:15.7138192Z [m-1]     @     0x7f4257530262  yb::rpc::OutboundCall::InvokeCallbackSync() (src/yb/rpc/outbound_call.cc:351)
2021-07-09T03:28:15.7139596Z [m-1]     @     0x7f4257530224  yb::rpc::InvokeCallbackTask::Run() (src/yb/rpc/outbound_call.cc:120)
2021-07-09T03:28:15.7140969Z [m-1]     @     0x7f42575afb0b  yb::rpc::(anonymous namespace)::Worker::Execute() (src/yb/rpc/thread_pool.cc:105)
2021-07-09T03:28:15.7144617Z [m-1]     @     0x7f42575b03ff  decltype(*(std::forward<yb::rpc::(anonymous namespace)::Worker*&>(fp0)).*fp()) std::__invoke<void (yb::rpc::(anonymous namespace)::Worker::*&)(), yb::rpc::(anonymous namespace)::Worker*&, void>(void (yb::rpc::(anonymous namespace)::Worker::*&)(), yb::rpc::(anonymous namespace)::Worker*&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/type_traits:3840)
2021-07-09T03:28:15.7151204Z [m-1]     @     0x7f42575b03cb  std::__bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), tuple<yb::rpc::(anonymous namespace)::Worker*>, tuple<>, __is_valid_bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), tuple<yb::rpc::(anonymous namespace)::Worker*>, tuple<> >::value>::type std::__apply_functor<void (yb::rpc::(anonymous namespace)::Worker::*)(), tuple<yb::rpc::(anonymous namespace)::Worker*>, 0ul, tuple<> >(void (yb::rpc::(anonymous namespace)::Worker::*&)(), tuple<yb::rpc::(anonymous namespace)::Worker*>&, std::__tuple_indices<0ul>, tuple<>&&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2853)
2021-07-09T03:28:15.7158146Z [m-1]     @     0x7f42575b03ac  std::__bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), tuple<yb::rpc::(anonymous namespace)::Worker*>, tuple<>, __is_valid_bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), tuple<yb::rpc::(anonymous namespace)::Worker*>, tuple<> >::value>::type std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>::operator()<>() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2886)
2021-07-09T03:28:15.7164569Z [m-1]     @     0x7f42575b0398  decltype(std::forward<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&>(fp)()) std::__invoke<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&>(std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/type_traits:3899)
2021-07-09T03:28:15.7170414Z [m-1]     @     0x7f42575b0378  void std::__invoke_void_return_wrapper<void>::__call<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&>(std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/__functional_base:348)
2021-07-09T03:28:15.7176134Z [m-1]     @     0x7f42575b0360  std::__function::__alloc_func<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>, std::allocator<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&> >, void ()>::operator()() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1557)
2021-07-09T03:28:15.7181762Z [m-1]     @     0x7f42575afdac  std::__function::__func<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>, std::allocator<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&> >, void ()>::operator()() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1731)
2021-07-09T03:28:15.7186413Z [m-1]     @     0x7f425891a722  std::__function::__value_func<void ()>::operator()() const (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1884)
2021-07-09T03:28:15.7190166Z [m-1]     @     0x7f425891a708  std::function<void ()>::operator()() const (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2556)
2021-07-09T03:28:15.7192588Z [m-1]     @     0x7f42563f711a  yb::Thread::SuperviseThread(void*) (src/yb/util/thread.cc:771)
2021-07-09T03:28:15.7193830Z [m-1]     @     0x7f4259267ea4 
2021-07-09T03:28:15.7194536Z [m-1]     @     0x7f4258f909fc 
2021-07-09T03:28:15.7195049Z [m-1] 
2021-07-09T03:28:15.7195644Z [m-1] *** Check failure stack trace: ***
2021-07-09T03:28:15.7196911Z [m-1]     @     0x7f4256345ee0  yb::(anonymous namespace)::DumpStackTraceAndExit() (src/yb/util/logging.cc:166)
2021-07-09T03:28:15.7197870Z [m-1]     @     0x7f42559a57aa 
2021-07-09T03:28:15.7198441Z [m-1]     @     0x7f42559a5da7 
2021-07-09T03:28:15.7199019Z [m-1]     @     0x7f42559a8f5e 
2021-07-09T03:28:15.7200129Z [m-1]     @     0x7f4256338def  yb::HdrHistogram::IncrementBy(long, long) (src/yb/util/hdr_histogram.cc:0)
2021-07-09T03:28:15.7201495Z [m-1]     @     0x7f4256338c2d  yb::HdrHistogram::Increment(long) (src/yb/util/hdr_histogram.cc:176)
2021-07-09T03:28:15.7202843Z [m-1]     @     0x7f425636602c  yb::Histogram::Increment(long) (src/yb/util/metrics.cc:885)
2021-07-09T03:28:15.7203928Z [m-1]     @     0x7f4258903abc  ~ReadRpc (src/yb/client/async_rpc.cc:705)
2021-07-09T03:28:15.7212052Z [m-1]     @     0x7f4258925bc5  std::__shared_ptr_emplace<yb::client::internal::ReadRpc, std::allocator<yb::client::internal::ReadRpc> >::__on_zero_shared() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/memory:3503)
2021-07-09T03:28:15.7217784Z [m-1]     @     0x7f42588fdb94  std::__shared_count::__release_shared() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/memory:3341)
2021-07-09T03:28:15.7221801Z [m-1]     @     0x7f42588fdb4d  std::__shared_weak_count::__release_shared() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/memory:3383)
2021-07-09T03:28:15.7225213Z [m-1]     @     0x7f4258904e01  ~shared_ptr (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/memory:4098)
2021-07-09T03:28:15.7228801Z [m-1]     @     0x7f425890530c  shared_ptr<yb::rpc::RpcCommand>::reset() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/memory:4233)
2021-07-09T03:28:15.7231313Z [m-1]     @     0x7f4258900fcb  yb::client::internal::AsyncRpc::Finished(yb::Status const&) (src/yb/client/async_rpc.cc:212)
2021-07-09T03:28:15.7235177Z [m-1]     @     0x7f425890e5a4  decltype(*(std::forward<yb::client::internal::ReadRpc*&>(fp0)).*fp(std::forward<yb::Status::OK&>(fp1))) std::__invoke<void (yb::client::internal::AsyncRpc::*&)(yb::Status const&), yb::client::internal::ReadRpc*&, yb::Status::OK&, void>(void (yb::client::internal::AsyncRpc::*&)(yb::Status const&), yb::client::internal::ReadRpc*&, yb::Status::OK&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/type_traits:3840)
2021-07-09T03:28:15.7242236Z [m-1]     @     0x7f425890e53a  std::__bind_return<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>, tuple<>, __is_valid_bind_return<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>, tuple<> >::value>::type std::__apply_functor<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>, 0ul, 1ul, tuple<> >(void (yb::client::internal::AsyncRpc::*&)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>&, std::__tuple_indices<0ul, 1ul>, tuple<>&&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2853)
2021-07-09T03:28:15.7249324Z [m-1]     @     0x7f425890e4e4  std::__bind_return<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>, tuple<>, __is_valid_bind_return<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), tuple<yb::client::internal::ReadRpc*, yb::Status::OK>, tuple<> >::value>::type std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>::operator()<>() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2886)
2021-07-09T03:28:15.7255895Z [m-1]     @     0x7f425890e4b0  decltype(std::forward<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>&>(fp)()) std::__invoke<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>&>(std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/type_traits:3899)
2021-07-09T03:28:15.7261869Z [m-1]     @     0x7f425890e480  void std::__invoke_void_return_wrapper<void>::__call<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>&>(std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/__functional_base:348)
2021-07-09T03:28:15.7267422Z [m-1]     @     0x7f425890e460  std::__function::__alloc_func<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>, std::allocator<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK> >, void ()>::operator()() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1557)
2021-07-09T03:28:15.7272958Z [m-1]     @     0x7f425890da4c  std::__function::__func<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK>, std::allocator<std::__bind<void (yb::client::internal::AsyncRpc::*)(yb::Status const&), yb::client::internal::ReadRpc*, yb::Status::OK> >, void ()>::operator()() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1731)
2021-07-09T03:28:15.7277511Z [m-1]     @     0x7f425891a722  std::__function::__value_func<void ()>::operator()() const (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1884)
2021-07-09T03:28:15.7281345Z [m-1]     @     0x7f425891a708  std::function<void ()>::operator()() const (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2556)
2021-07-09T03:28:15.7283855Z [m-1]     @     0x7f4257530262  yb::rpc::OutboundCall::InvokeCallbackSync() (src/yb/rpc/outbound_call.cc:351)
2021-07-09T03:28:15.7285268Z [m-1]     @     0x7f4257530224  yb::rpc::InvokeCallbackTask::Run() (src/yb/rpc/outbound_call.cc:120)
2021-07-09T03:28:15.7286652Z [m-1]     @     0x7f42575afb0b  yb::rpc::(anonymous namespace)::Worker::Execute() (src/yb/rpc/thread_pool.cc:105)
2021-07-09T03:28:15.7290306Z [m-1]     @     0x7f42575b03ff  decltype(*(std::forward<yb::rpc::(anonymous namespace)::Worker*&>(fp0)).*fp()) std::__invoke<void (yb::rpc::(anonymous namespace)::Worker::*&)(), yb::rpc::(anonymous namespace)::Worker*&, void>(void (yb::rpc::(anonymous namespace)::Worker::*&)(), yb::rpc::(anonymous namespace)::Worker*&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/type_traits:3840)
2021-07-09T03:28:15.7298837Z [m-1]     @     0x7f42575b03cb  std::__bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), tuple<yb::rpc::(anonymous namespace)::Worker*>, tuple<>, __is_valid_bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), tuple<yb::rpc::(anonymous namespace)::Worker*>, tuple<> >::value>::type std::__apply_functor<void (yb::rpc::(anonymous namespace)::Worker::*)(), tuple<yb::rpc::(anonymous namespace)::Worker*>, 0ul, tuple<> >(void (yb::rpc::(anonymous namespace)::Worker::*&)(), tuple<yb::rpc::(anonymous namespace)::Worker*>&, std::__tuple_indices<0ul>, tuple<>&&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2853)
2021-07-09T03:28:15.7307936Z [m-1]     @     0x7f42575b03ac  std::__bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), tuple<yb::rpc::(anonymous namespace)::Worker*>, tuple<>, __is_valid_bind_return<void (yb::rpc::(anonymous namespace)::Worker::*)(), tuple<yb::rpc::(anonymous namespace)::Worker*>, tuple<> >::value>::type std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>::operator()<>() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2886)
2021-07-09T03:28:15.7314327Z [m-1]     @     0x7f42575b0398  decltype(std::forward<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&>(fp)()) std::__invoke<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&>(std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/type_traits:3899)
2021-07-09T03:28:15.7320205Z [m-1]     @     0x7f42575b0378  void std::__invoke_void_return_wrapper<void>::__call<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&>(std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>&) (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/__functional_base:348)
2021-07-09T03:28:15.7325751Z [m-1]     @     0x7f42575b0360  std::__function::__alloc_func<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>, std::allocator<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&> >, void ()>::operator()() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1557)
2021-07-09T03:28:15.7331388Z [m-1]     @     0x7f42575afdac  std::__function::__func<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&>, std::allocator<std::__bind<void (yb::rpc::(anonymous namespace)::Worker::* const&)(), yb::rpc::(anonymous namespace)::Worker* const&> >, void ()>::operator()() (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1731)
2021-07-09T03:28:15.7336101Z [m-1]     @     0x7f425891a722  std::__function::__value_func<void ()>::operator()() const (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:1884)
2021-07-09T03:28:15.7339845Z [m-1]     @     0x7f425891a708  std::function<void ()>::operator()() const (/opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210706201111-a8bf7ab880-centos7-clang11/installed/uninstrumented/libcxx/include/c++/v1/functional:2556)
2021-07-09T03:28:15.7342293Z [m-1]     @     0x7f42563f711a  yb::Thread::SuperviseThread(void*) (src/yb/util/thread.cc:771)
2021-07-09T03:28:15.7343124Z [m-1]     @     0x7f4259267ea4 
2021-07-09T03:28:15.7343697Z [m-1]     @     0x7f4258f909fc 
void HdrHistogram::IncrementBy(int64_t value, int64_t count) {
  DCHECK_GE(value, 0); // <-- FAILED HERE: existing value is already less than 0.
  DCHECK_GE(count, 0);

  // Dissect the value into bucket and sub-bucket parts, and derive index into
  // counts array:
`

Could this be because HdrHistogram has already been destroyed by the time the IncrementBy call is happening?

karan-yb commented 2 years ago

Ran into same DCHECK during local testing on mac.

F20220802 20:21:29 ../../src/yb/util/hdr_histogram.cc:181] Check failed: value >= 0 (-3006 vs. 0)    
  2     @        0x10a09a510  google::LogDestination::LogToSinks()                                       
  3     @        0x10a099648  google::LogMessage::SendToLog()                                            
  4     @        0x10a09a004  google::LogMessage::Flush()                                                
  5     @        0x10a09e038  google::LogMessageFatal::~LogMessageFatal()                                
  6     @        0x10a09af54  google::LogMessageFatal::~LogMessageFatal()                                
  7     @        0x116f7591c  yb::HdrHistogram::IncrementBy()                                            
  8     @        0x116f75860  yb::HdrHistogram::Increment()                                              
  9     @        0x116ff8b18  yb::Histogram::Increment()                                                 
 10     @        0x112172230  rocksdb::StatisticsMetricImpl::measureTime()                               
 11     @        0x111b9a778  rocksdb::StopWatch::~StopWatch()                                           
 12     @        0x111b6608c  rocksdb::StopWatch::~StopWatch()                                           
 13     @        0x111b6e704  rocksdb::DBImpl::WriteImpl()                                               
 14     @        0x111b6c480  rocksdb::DBImpl::Write()                                                   
 15     @        0x10634c91c  yb::tablet::Tablet::WriteToRocksDB()                                       
 16     @        0x10634b8fc  yb::tablet::Tablet::ApplyKeyValueRowOperations()                           
 17     @        0x10634b024  yb::tablet::Tablet::ApplyOperation()                                       
 18     @        0x10634a9e4  yb::tablet::Tablet::ApplyRowOperations()                                   
 19     @        0x1063fe968  yb::tablet::TabletBootstrap::PlayWriteRequest()                            
 20     @        0x1063fdd1c  yb::tablet::TabletBootstrap::PlayAnyRequest()                              
 21     @        0x1063fca38  yb::tablet::TabletBootstrap::MaybeReplayCommittedEntry()                   
 22     @        0x1063f3afc  yb::tablet::TabletBootstrap::ApplyCommittedPendingReplicates()             
 23     @        0x1063f8184  yb::tablet::TabletBootstrap::HandleReplicateMessage()                      
 24     @        0x1063f2d40  yb::tablet::TabletBootstrap::HandleEntry()                                 
 25     @        0x1063eb4e0  yb::tablet::TabletBootstrap::PlaySegments()                                
 26     @        0x1063e32a0  yb::tablet::TabletBootstrap::Bootstrap()                                   
 27     @        0x1063e2024  yb::tablet::BootstrapTabletImpl()                                          
 28     @        0x106415b80  yb::tablet::BootstrapTablet()                                              
 29     @        0x103f4cd10  yb::tserver::TSTabletManager::OpenTablet()                                 
 30     @        0x103f84948  _ZNSt3__1L8__invokeIRMN2yb7tserver15TSTabletManagerEFvRK13scoped_refptrINS1_6tablet17RaftGroupMetadataEERKS4_INS2_27TransitionInProgressDeleterEEERPS3_JRS7_RSB_EvEEDTcldsdescT0_fp0_fp_spscT1_fp1_EEOT_OSL_DpOSM_
 31     @        0x103f848bc  _ZNSt3__1L15__apply_functorIMN2yb7tserver15TSTabletManagerEFvRK13scoped_refptrINS1_6tablet17RaftGroupMetadataEERKS4_INS2_27TransitionInProgressDeleterEEENS_5tupleIJPS3_S7_SB_EEEJLm0ELm1ELm2EENSG_IJEEEEENS_13__bind_r
 32     @        0x103f84844  _ZNSt3__16__bindIMN2yb7tserver15TSTabletManagerEFvRK13scoped_refptrINS1_6tablet17RaftGroupMetadataEERKS4_INS2_27TransitionInProgressDeleterEEEJPS3_S9_RSB_EEclIJEEENS_13__bind_returnISF_NS_5tupleIJSG_S7_SB_EEENSL_IJD
 33     @        0x103f845e0  yb::RunnableImpl<>::Run()