fastio / 1store

NoSQL data store using the SEASTAR framework, compatible with Redis
GNU Affero General Public License v3.0
1.31k stars 170 forks source link

pedis crashed after memtier_benchmark restart #32

Open tracymacding opened 6 years ago

tracymacding commented 6 years ago

backtrace: `

0 0x000000000074986c in boost::intrusive::list_node_traits<void*>::get_previous (n=@0x7fffffff96c0: 0x8e)

at /opt/scylladb/include/boost/intrusive/detail/list_node.hpp:54

1 0x00000000007dc731 in boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*> >::transfer (p=@0x7fffffff96c0: 0x8e,

i=@0x7fffffff96b0: 0x302bddb67028) at /opt/scylladb/include/boost/intrusive/circular_list_algorithms.hpp:284

2 0x00000000007d6e7b in boost::intrusive::list_impl<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1u>, unsigned long, false, void>::splice (this=0x30000027bb30, p=..., x=..., new_ele=...)

at /opt/scylladb/include/boost/intrusive/list.hpp:928

3 0x00000000007d2542 in log_histogram<logalloc::segment_descriptor, logalloc::segment_descriptor_hist_options>::adjust_up (this=0x30000027b240, v=...)

at ./utils/log_histogram.hh:237

4 0x00000000007cf8de in logalloc::region_impl::free (this=0x30000027b210, obj=0x302bd806ad70, size=24) at utils/logalloc.cc:1354

5 0x0000000000a3fdb1 in allocation_strategy::destroy<managed > (this=0x30000027b210, obj=0x302bd806ad70)

at ./utils/allocation_strategy.hh:151

6 0x0000000000a393f2 in managed_ref::~managed_ref (this=0x302bd806ad08, __in_chrg=) at utils/managed_ref.hh:61

7 0x0000000000a34470 in redis::cache_entry::~cache_entry (this=0x302bd806acf0, __in_chrg=) at cache.hh:179

8 0x0000000000a3a1b5 in allocation_strategy::destroy (this=0x30000027b210, obj=0x302bd806acf0) at ./utils/allocation_strategy.hh:150

9 0x0000000000a34ed4 in auto current_deleter()::{lambda(redis::cache_entry)#1}::operator()(redis::cache_entry) const (

__closure=0x7fffffff99e0, obj=0x302bd806acf0) at ./utils/allocation_strategy.hh:202

10 0x0000000000a4dd05 in boost::intrusive::detail::node_disposer<auto current_deleter()::{lambda(redis::cache_entry)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>::operator()(boost::intrusive::slist_node<void>* const&) (this=0x7fffffff99e0, p=@0x7fffffff9938: 0x302bd806acf8)

at /opt/scylladb/include/boost/intrusive/detail/node_cloner_disposer.hpp:96

11 0x0000000000a4bb9c in boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter()::{lambda(redis::cache_entry)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void>, boost::intrusive::slist_node<void> >::operator()(boost::intrusive::slist_node<void*>) (this=0x7fffffff99e0, to_clone=0x302bd806acf8) at /opt/scylladb/include/boost/intrusive/hashtable.hpp:570

12 0x0000000000a47be4 in boost::intrusive::slist_impl<boost::intrusive::trivial_value_traits<boost::intrusive::slist_node_traits<void>, (boost::intrusive::link_mode_type)0>, unsigned long, 0ul, void>::erase_after_and_dispose<boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter()::{lambda(redis::cache_entry)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void>, boost::intrusive::slist_node<void> > >(boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter()::{lambda(redis::cache_entry)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void>, boost::intrusive::slist_node<void> ><boost::intrusive::trivial_value_traits<boost::intrusive::slist_node_traits<void>, (boost::intrusive::link_mode_type)0>, true>, boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter()::{lambda(redis::cache_entry)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void>, boost::intrusive::slist_node<void> >) (this=0x302bde053610, prev=..., disposer=...) at /opt/scylladb/include/boost/intrusive/slist.hpp:1053

13 0x0000000000a41417 in boost::intrusive::bucket_plus_vtraits<boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, boost::intrusive::detail::bucket_traits_impl<boost::intrusive::detail::get_slist_impl<boost::intrusive::slist_node_traits<void> >::type> >::priv_erase_node<boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter()::{lambda(redis::cache_entry)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void>, boost::intrusive::slist_node<void> > >(boost::intrusive::detail::bucket_impl<boost::intrusive::detail::get_slist_impl<boost::intrusive::slist_node_traits<void> >::type>&, boost::intrusive::slist_ite---Type to continue, or q to quit---

rator<boost::intrusive::trivial_value_traits<boost::intrusive::slist_node_traits<void>, (boost::intrusive::link_mode_type)0>, false>, boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter()::{lambda(redis::cache_entry)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void>, boost::intrusive::slist_node<void*> >, boost::movedetail::bool) (b=..., i=..., node_disposer=...) at /opt/scylladb/include/boost/intrusive/hashtable.hpp:757

14 0x0000000000a3a5f6 in boost::intrusive::hashtable_impl<boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, void, void, void, boost::intrusive::detail::bucket_traits_impl<boost::intrusive::detail::get_slist_impl<boost::intrusive::slist_node_traits<void> >::type>, unsigned long, 7ul>::erase_and_dispose<auto current_deleter()::{lambda(redis::cache_entry)#1}>(boost::intrusive::bucket_plus_vtraits<boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, boost::intrusive::detail::bucket_traits_impl<boost::intrusive::detail::get_slist_impl<boost::intrusive::slist_node_traits<void*> >::type> >, boost::move_detail::disable_if_convertible) (this=0x30000023a430, i=..., disposer=...)

at /opt/scylladb/include/boost/intrusive/hashtable.hpp:2279

15 0x0000000000a35490 in redis::cache::insert_if (this=0x30000023a420, entry=0x302bd81e8060, expired=0, nx=false, xx=false) at cache.hh:513

16 0x00000000009fa9e4 in redis::database::<lambda()>::operator()(void) const (__closure=0x7fffffff9c60) at db.cc:290

17 0x00000000009faab6 in with_allocator<redis::database::set(redis::redis_key, bytes, long int, uint32_t)::<lambda()> >(allocation_strategy &, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4ce4579, DIE 0x4e85566>) (alloc=...,

func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4ce4579, DIE 0x4e85566>) at ./utils/allocation_strategy.hh:271

18 0x00000000009fabbc in redis::database::set (this=0x30000023a410, rk=..., val=..., expired=0, flag=1) at db.cc:298

19 0x00000000008d0b3a in std::invoke_impl<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > >, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::* const&)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), seastar::shared_ptr, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>(std::invoke_memfun_deref, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::* const&)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), seastar::shared_ptr&&, redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&&, unsigned char&&) (__f=

@0x7fffffffa030: (seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::database * const, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31, true>, long, unsigned int)) 0x9faaea <redis::database::set(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, 
__t=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4a8ab08>, 
__args#0=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4a8ab33>, 
__args#1=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4a8ab42>, 
__args#2=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4a8ab51>, 
__args#3=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4a8ab60>)
at /opt/scylladb/include/c++/7/bits/invoke.h:73

20 0x00000000008ab9e1 in std::invoke<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database:: const&)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), seastar::shared_ptr, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>(seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database:: const&)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), seastar::shared_ptr&&, redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&&, unsigned char&&) (fn=

@0x7fffffffa030: (seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::database * const, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31, true>, long, unsigned int)) 0x9faaea <redis::database::set(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, 
__args#0=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab6312>, 

---Type to continue, or q to quit--- args#1=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab6321>, args#2=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab6330>, args#3=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab633f>, args#4=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab634e>) at /opt/scylladb/include/c++/7/bits/invoke.h:96

21 0x00000000008998b7 in std::_Mem_fn_base<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), true>::operator()<seastar::shared_ptr, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>(seastar::shared_ptr&&, redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&&, unsigned char&&) const (this=0x7fffffffa030,

__args#0=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada3ed>, 
__args#1=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada3fc>, 
__args#2=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada40b>, 
__args#3=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada41a>, 
__args#4=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada429>)
at /opt/scylladb/include/c++/7/functional:175

22 0x00000000008871c3 in seastar::apply_helper<std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, std::tuple<seastar::shared_ptr, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>&&, std::integer_sequence<unsigned long, 0ul, 1ul, 2ul, 3ul, 4ul> >::apply(std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>&&, std::tuple<seastar::shared_ptr, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>&&) (

func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd5f9>, 
args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd608>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:35

23 0x000000000088721f in seastar::apply<std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, seastar::shared_ptr, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>(std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>&&, std::tuple<seastar::shared_ptr, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>&&) (

func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd5bc>, 
args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd5ca>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:43

24 0x000000000088726f in seastar::futurize<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > >::apply<std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, seastar::shared_ptr, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>(std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>&&, std::tuple<seastar::shared_ptr, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>&&) (

func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd557>, 
args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd566>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:1389

25 0x000000000087d1ea in seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > seastar::sharded---Type to continue, or q to quit---

::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)::{lambda()#1}::operator()() (__closure=0x30000010d310) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/sharded.hh:317

26 0x00000000008998ef in seastar::futurize<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > >::apply<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > seastar::sharded::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)::{lambda()#1}&>(seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > seastar::sharded::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)::{lambda()#1}&) (func=...) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:1399

27 0x000000000088737b in seastar::smp::submit_to<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > seastar::sharded::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)::{lambda()#1}>(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > seastar::sharded::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)::{lambda()#1}&&) (t=0,

func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab5e5a>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/reactor.hh:1258

28 0x000000000087d34e in seastar::sharded::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)

(this=0x1218c20 <redis::_databases>, id=0, func=
(seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::database * const, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31, true>, long, unsigned int)) 0x9faaea <redis::database::set(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, 

---Type to continue, or q to quit--- args#0=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada2f4>, args#1=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada304>, args#2=@0x7fffffffa278: 0, args#3=@0x7fffffffa277: 1 '\001') at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/sharded.hh:318

29 0x0000000000829d02 in redis::redis_service::set (this=0x2aaaaaae1e48, req=...) at redis.cc:135

30 0x000000000099ff7b in redis::<lambda(redis::request_wrapper&)>::operator()(redis::request_wrapper &) const (__closure=0x30000019eb18, req=...)

at server.cc:34

31 0x00000000009af419 in std::_Function_handler<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > >(redis::request_wrapper&), redis::<lambda(redis::request_wrapper&)> >::_M_invoke(const std::_Any_data &, redis::request_wrapper &) (functor=..., args#0=...)

at /opt/scylladb/include/c++/7/bits/std_function.h:302

32 0x00000000009d45d5 in std::function<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > (redis::request_wrapper&)>::operator()(redis::request_wrapper&) const (this=0x30000019eb18, __args#0=...) at /opt/scylladb/include/c++/7/bits/std_function.h:706

33 0x00000000009a4d5d in redis::server::connection::do_handle_one (this=0x3000000c6208, req=...) at server.cc:156

34 0x00000000009a4de7 in redis::server::connection::<lambda()>::<lambda()>::operator()(void) const (__closure=0x7fffffffa6e0) at server.cc:174

35 0x00000000009b6722 in seastar::apply_helper<redis::server::connection::handle()::<lambda()>::<lambda()>, std::tuple<>&&, std::integer_sequence >::apply(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf097>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf0a5>) (

func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf097>, 
args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf0a5>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:35

36 0x00000000009b676d in seastar::apply<redis::server::connection::handle()::<lambda()>::<lambda()> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf05a>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf068>) (func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf05a>,

args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf068>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:43

37 0x00000000009b67bd in seastar::futurize<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > >::apply<redis::server::connection::handle()::<lambda()>::<lambda()> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf013>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf022>) (

func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf013>, 
args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf022>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:1389

38 0x00000000009aa256 in seastar::future<>::then<redis::server::connection::handle()::<lambda()>::<lambda()> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4c5b31d>) (this=0x7fffffffa6d0,

func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4c5b31d>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:952

39 0x00000000009a4e41 in redis::server::connection::<lambda()>::operator()(void) const (__closure=0x300000104188) at server.cc:175

40 0x00000000009b699f in seastar::apply_helper<redis::server::connection::handle()::<lambda()>, std::tuple<>&&, std::integer_sequence >::apply(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbed0b>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbed19>) (

func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbed0b>, 
args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbed19>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:35

41 0x00000000009b69ea in seastar::apply<redis::server::connection::handle()::<lambda()> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/relea---Type to continue, or q to quit---

se/pedis, CU 0x4b1e702, DIE 0x4cbecce>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbecdc>) ( func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbecce>, args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbecdc>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:43

42 0x00000000009b6a3b in seastar::futurize<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > > >::apply<redis::server::connection::handle()::<lambda()> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbec87>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbec96>) (

func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbec87>, 
args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbec96>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:1389

43 0x00000000009d1573 in seastar::future<>::<lambda(auto:1&&)>::operator()<seastar::future_state<> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4c5b9fe>) (__closure=0x300000104158,

state=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4c5b9fe>)
at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:963

44 0x00000000009d16d6 in seastar::continuation<seastar::future::then(Func&&) [with Func = redis::server::connection::handle()::<lambda()>; Result = seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message > > >; T = {}]::<lambda(auto:1&&)> >::run_and_dispose(void) (

this=0x300000104140) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:414

45 0x0000000000472a5f in seastar::reactor::run_tasks (this=this@entry=0x300000020000, tq=...) at core/reactor.cc:2699

46 0x0000000000472d53 in seastar::reactor::run_some_tasks (this=this@entry=0x300000020000) at core/reactor.cc:3122

47 0x00000000004c6ed0 in run_some_tasks (this=0x300000020000) at /u01/jeff.dk/github/tracymacding/pedis/seastar/util/log.hh:149

48 seastar::reactor::run (this=0x300000020000) at core/reactor.cc:3269

49 0x0000000000568d4e in seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) (this=, ac=,

av=0x7fffffffb1f8, func=<optimized out>) at core/app-template.cc:185

50 0x0000000000730603 in main (ac=3, av=0x7fffffffb1f8) at main.cc:81

`

tracymacding commented 6 years ago

from debug info, program cores here: `

0 0x0000000000a3ed1d in boost::intrusive::list_node_traits<void*>::get_previous (n=@0x7fffffff96c0: 0x8e)

at /opt/scylladb/include/boost/intrusive/detail/list_node.hpp:51

1

2 0x000000000074986c in boost::intrusive::list_node_traits<void*>::get_previous (n=@0x7fffffff96c0: 0x8e)

at /opt/scylladb/include/boost/intrusive/detail/list_node.hpp:54

3 0x00000000007dc731 in boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*> >::transfer (p=@0x7fffffff96c0: 0x8e,

i=@0x7fffffff96b0: 0x302bddb67028) at /opt/scylladb/include/boost/intrusive/circular_list_algorithms.hpp:284

4 0x00000000007d6e7b in boost::intrusive::list_impl<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1u>, unsigned long, false, void>::splice (this=0x30000027bb30, p=..., x=..., new_ele=...)

at /opt/scylladb/include/boost/intrusive/list.hpp:928

5 0x00000000007d2542 in log_histogram<logalloc::segment_descriptor, logalloc::segment_descriptor_hist_options>::adjust_up (this=0x30000027b240, v=...)

at ./utils/log_histogram.hh:237

`

in function adjust_up: void adjust_up(T& v) { auto b = traits::cached_bucket(v); auto nb = opts.bucket_of(traits::hist_key(v)); if (nb != b) { traits::cache_bucket(v, nb); _buckets[nb].splice(_buckets[nb].begin(), _buckets[b], _buckets[b].iterator_to(v)); _watermark = std::max(ssize_t(nb), _watermark); } } gdb shows variables value (gdb) p nb $35 = 143 (gdb) p b $36 = 142 (gdb) p _buckets[nb].begin() $37 = {static stateful_value_traits = false, members_ = {nodeptr_ = 0x30000027bb30}} (gdb) p _buckets[b] $38 = (std::array<boost::intrusive::list<logalloc::segment_descriptor, boost::intrusive::constant_time_size<false>, void, void, void>, 143>::value_type &) @0x30000027bb20: {<boost::intrusive::list_impl<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1>, unsigned long, false, void>> = {static constant_time_size = false, static stateful_value_traits = <optimized out>, static has_container_from_iterator = <optimized out>, static safemode_or_autounlink = true, data_ = {<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1>> = {<boost::intrusive::bhtraits_base<logalloc::segment_descriptor, boost::intrusive::list_node<void*>*, boost::intrusive::dft_tag, 1>> = {<No data fields>}, static link_mode = boost::intrusive::safe_link}, root_plus_size_ = {<boost::intrusive::detail::size_holder<false, unsigned long, void>> = {static constant_time_size = <optimized out>}, m_header = {<boost::intrusive::list_node<void*>> = {next_ = 0x302bddb67028, prev_ = 0x302bddb67028}, <No data fields>}}}}, <No data fields>} (gdb) p _buckets[b].iterator_to(v) $39 = {static stateful_value_traits = false, members_ = {nodeptr_ = 0x30000027bb20}}

while in boost splice: ` (gdb) f 4

4 0x00000000007d6e7b in boost::intrusive::list_impl<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1u>, unsigned long, false, void>::splice (this=0x30000027bb30, p=..., x=..., new_ele=...)

at /opt/scylladb/include/boost/intrusive/list.hpp:928

928 node_algorithms::transfer(p.pointed_node(), new_ele.pointed_node()); (gdb) p p $40 = {static stateful_valuetraits = false, members = {nodeptr_ = 0x8e}} (gdb) p x $41 = (boost::intrusive::list_impl<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1>, unsigned long, false, void> &) @0x30000027bb20: {static constant_time_size = false, static stateful_value_traits = , static has_container_from_iterator = , static safemode_orautounlink = true, data = {<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1>> = {<boost::intrusive::bhtraits_base<logalloc::segment_descriptor, boost::intrusive::list_node<void>, boost::intrusive::dft_tag, 1>> = {}, static link_mode = boost::intrusive::safe_link}, root_plussize = {<boost::intrusive::detail::size_holder<false, unsigned long, void>> = {static constant_time_size = }, m_header = {<boost::intrusive::listnode<void*>> = {next = 0x302bddb67028, prev_ = 0x302bddb67028}, }}}} (gdb) p new_ele $42 = {static stateful_valuetraits = false, members = {nodeptr_ = 0x302bddb67028}} `

it seems the first parameter @p in function splice contains invalid members_(0x8e) which caused the crash.