search

sirikata / sirikata

Sirikata is a BSD-licensed platform for networked 3d environments
http://www.sirikata.com/
Other
126 stars 39 forks source link

Destruction of SST Connections is not thread safe #508

Closed ewencp closed 12 years ago

ewencp commented 12 years ago

I'm sometimes getting a crash from two threads destroying different SST connections at the same time on the space server.

Stack trace from first thread:

[Switching to thread 1 (Thread 0x7ffff7fc0720 (LWP 6508))]#0  0x00007ffff4f5d656 in std::_Rb_tree_rotate_right(std::_Rb_tree_node_base*, std::_Rb_tree_node_base*&) () from /usr/lib/libstdc++.so.6
(gdb) bt
#0  0x00007ffff4f5d656 in std::_Rb_tree_rotate_right(std::_Rb_tree_node_base*, std::_Rb_tree_node_base*&) () from /usr/lib/libstdc++.so.6
#1  0x00007ffff4f5d967 in std::_Rb_tree_rebalance_for_erase(std::_Rb_tree_node_base*, std::_Rb_tree_node_base&) () from /usr/lib/libstdc++.so.6
#2  0x00007ffff5fe6e95 in std::_Rb_tree<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID>, std::pair<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> const, Sirikata::OHDP::Port*>, std::_Select1st<std::pair<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> const, Sirikata::OHDP::Port*> >, std::less<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> >, std::allocator<std::pair<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> const, Sirikata::OHDP::Port*> > >::erase (this=0x7fffe4009920, __position=...) at /usr/include/c++/4.4/bits/stl_tree.h:1347
#3  0x00007ffff5fe312f in std::map<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID>, Sirikata::OHDP::Port*, std::less<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> >, std::allocator<std::pair<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> const, Sirikata::OHDP::Port*> > >::erase (this=0x7fffe4009920, __position=...)
    at /usr/include/c++/4.4/bits/stl_map.h:567
#4  0x00007ffff5fe1d0f in Sirikata::SST::BaseDatagramLayer<Sirikata::OHDP::SpaceNodeID>::unlisten (this=0x7fffe4009910, ep=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/ohdp/SST.hpp:98
#5  0x00007ffff60052dc in Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::finalCleanup (this=0x7fffe4042340) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/network/SSTImpl.hpp:1136
#6  0x00007ffff5ffebbb in Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::~Connection (this=0x7fffe4042340, __in_chrg=<value optimized out>) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/network/SSTImpl.hpp:1243
#7  0x00007ffff6012aa2 in std::tr1::_Sp_deleter<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >::operator() (this=0x7fffe403a808, __p=0x7fffe4042340) at /usr/include/c++/4.4/tr1/shared_ptr.h:97
#8  0x00007ffff6011cdf in std::tr1::_Sp_counted_base_impl<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::_Sp_deleter<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >, (__gnu_cxx::_Lock_policy)2>::_M_dispose (this=0x7fffe403a7f0) at /usr/include/c++/4.4/tr1/shared_ptr.h:75
#9  0x00007ffff5f70952 in std::tr1::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release (this=0x7fffe403a7f0) at /usr/include/c++/4.4/tr1_impl/boost_sp_counted_base.h:140
#10 0x00007ffff5f6fbf1 in std::tr1::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count (this=0x7fffdc0278e8, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/shared_ptr.h:153
#11 0x00007ffff5fe332c in std::tr1::__shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=0x7fffdc0278e0, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/shared_ptr.h:358
#12 0x00007ffff5fe336c in std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >::~shared_ptr (this=0x7fffdc0278e0, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/shared_ptr.h:834
#13 0x00007ffff5fe8174 in std::tr1::_Tuple_impl<1, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> > >::~_Tuple_impl (this=0x7fffdc0278e0, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/tuple:70
#14 0x00007ffff5fe818e in std::tr1::_Tuple_impl<0, Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> > >::~_Tuple_impl (this=0x7fffdc0278e0, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/tuple:70
#15 0x00007ffff5fe81a8 in std::tr1::tuple<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> > >::~tuple (this=0x7fffdc0278e0, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/tuple:129
#16 0x00007ffff5fe81c6 in std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::*)(std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)>(Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)>::~_Bind(void) (
    this=0x7fffdc0278d0, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1_impl/functional:1115
#17 0x00007ffff5fff893 in _Base_manager<std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::*)(std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)>(Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)> >::_M_destroy(std::tr1::_Any_data &, std::tr1::false_type) (__victim=...) at /usr/include/c++/4.4/tr1_impl/functional:1523
#18 0x00007ffff5ff5370 in _Base_manager<std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::*)(std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)>(Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)> >::_M_manager(std::tr1::_Any_data &, const std::tr1::_Any_data &, std::tr1::_Manager_operation) (__dest=..., __source=..., __op=std::tr1::__destroy_functor) at /usr/include/c++/4.4/tr1_impl/functional:1547
#19 0x00007ffff66e545b in std::tr1::_Function_base::~_Function_base (this=Could not find the frame base for "std::tr1::_Function_base::~_Function_base()".
) at /usr/include/c++/4.4/tr1_impl/functional:1628
#20 0x00007ffff66e7f9a in std::tr1::function<void()>::~function(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1_impl/functional:1463
#21 0x00007ffff680ef7f in boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, std::tr1::function<void()> >::~wrapped_handler(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/wrapped_handler.hpp:34
#22 0x00007ffff680f687 in _Base_manager<boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, std::tr1::function<void()> > >::_M_destroy(std::tr1::_Any_data &, std::tr1::false_type) (__victim=DWARF-2 expression error: DW_OP_reg operations must be used either alone or in conjuction with DW_OP_piece or DW_OP_bit_piece.
) at /usr/include/c++/4.4/tr1_impl/functional:1523
#23 0x00007ffff680f44c in _Base_manager<boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, std::tr1::function<void()> > >::_M_manager(std::tr1::_Any_data &, const std::tr1::_Any_data &, std::tr1::_Manager_operation) (__dest=Could not find the frame base for "_Base_manager<boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, std::tr1::function<void()> > >::_M_manager(std::tr1::_Any_data &, const std::tr1::_Any_data &, std::tr1::_Manager_operation)".
) at /usr/include/c++/4.4/tr1_impl/functional:1547
#24 0x00007ffff66e545b in std::tr1::_Function_base::~_Function_base (this=Could not find the frame base for "std::tr1::_Function_base::~_Function_base()".
) at /usr/include/c++/4.4/tr1_impl/functional:1628
#25 0x00007ffff66e7f9a in std::tr1::function<void()>::~function(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1_impl/functional:1463
#26 0x00007ffff6773d2e in std::tr1::_Tuple_impl<2, std::tr1::function<void()> >::~_Tuple_impl(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1/tuple:70
#27 0x00007ffff6807acb in std::tr1::_Tuple_impl<1, std::tr1::shared_ptr<boost::asio::basic_deadline_timer<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime>, boost::asio::deadline_timer_service<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime> > > >, std::tr1::function<void()> >::~_Tuple_impl(void) (this=Could not find the frame base for "std::tr1::_Tuple_impl<1, std::tr1::shared_ptr<boost::asio::basic_deadline_timer<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime>, boost::asio::deadline_timer_service<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime> > > >, std::tr1::function<void()> >::~_Tuple_impl(void)".
)
    at /usr/include/c++/4.4/tr1/tuple:70
#28 0x00007ffff6807b0c in std::tr1::_Tuple_impl<0, std::tr1::_Placeholder<1>, std::tr1::shared_ptr<boost::asio::basic_deadline_timer<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime>, boost::asio::deadline_timer_service<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime> > > >, std::tr1::function<void()> >::~_Tuple_impl(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1/tuple:70
#29 0x00007ffff6807b26 in std::tr1::tuple<std::tr1::_Placeholder<1>, std::tr1::shared_ptr<boost::asio::basic_deadline_timer<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime>, boost::asio::deadline_timer_service<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime> > > >, std::tr1::function<void()> >::~tuple(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1/tuple:129
#30 0x00007ffff6807b44 in std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, std::tr1::shared_ptr<boost::asio::basic_deadline_timer<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime>, boost::asio::deadline_timer_service<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime> > > >, std::tr1::function<void()>))(const boost::system::error_code&, const Sirikata::Network::deadline_timer_ptr&, const Sirikata::Network::IOCallback&)>::~_Bind(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1_impl/functional:1115
#31 0x00007ffff680b2d0 in boost::asio::detail::binder1<std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, std::tr1::shared_ptr<boost::asio::basic_deadline_timer<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime>, boost::asio::deadline_timer_service<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime> > > >, std::tr1::function<void()>))(const boost::system::error_code&, const Sirikata::Network::deadline_timer_ptr&, const Sirikata::Network::IOCallback&)>, boost::system::error_code>::~binder1(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/bind_handler.hpp:29
#32 0x00007ffff680c0d7 in handler_wrapper<boost::asio::detail::binder1<std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, std::tr1::shared_ptr<boost::asio::basic_deadline_timer<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime>, boost::asio::deadline_timer_service<boost::posix_time::ptime, boost::asio::time_traits<boost::posix_time::ptime> > > >, std::tr1::function<void()>))(const boost::system::error_code&, const Sirikata::Network::deadline_timer_ptr&, const Sirikata::Network::IOCallback&)>, boost::system::error_code> >::do_call(boost::asio::detail::handler_queue::handler *) (base=Asked for position 0 of stack, stack only has 0 elements on it.
)
    at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/handler_queue.hpp:192
#33 0x00007ffff67859b9 in boost::asio::detail::handler_queue::handler::invoke (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/handler_queue.hpp:39
#34 0x00007ffff678bfdf in boost::asio::detail::task_io_service<boost::asio::detail::epoll_reactor<false> >::do_one (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/task_io_service.hpp:268
#35 0x00007ffff67889f7 in boost::asio::detail::task_io_service<boost::asio::detail::epoll_reactor<false> >::run (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/task_io_service.hpp:103
#36 0x00007ffff6785ed9 in boost::asio::io_service::run (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/impl/io_service.ipp:68
#37 0x00007ffff68070b3 in Sirikata::Network::IOService::run (this=Unhandled dwarf expression opcode 0x0
) at /home/ewencp/sirikata.sirikata/libcore/src/network/IOService.cpp:118
#38 0x00007ffff6824f7d in Sirikata::Context::run (this=Unhandled dwarf expression opcode 0x0
) at /home/ewencp/sirikata.sirikata/libcore/src/service/Context.cpp:106
#39 0x00000000005fa7ea in main (argc=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/space/src/main.cpp:372

and from the second:

[Switching to thread 13 (Thread 0x7fffe8d1b700 (LWP 6526))]#0  0x00007ffff4f5daeb in std::_Rb_tree_rebalance_for_erase(std::_Rb_tree_node_base*, std::_Rb_tree_node_base&) () from /usr/lib/libstdc++.so.6
(gdb) bt
#0  0x00007ffff4f5daeb in std::_Rb_tree_rebalance_for_erase(std::_Rb_tree_node_base*, std::_Rb_tree_node_base&) () from /usr/lib/libstdc++.so.6
#1  0x00007ffff5fe6e95 in std::_Rb_tree<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID>, std::pair<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> const, Sirikata::OHDP::Port*>, std::_Select1st<std::pair<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> const, Sirikata::OHDP::Port*> >, std::less<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> >, std::allocator<std::pair<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> const, Sirikata::OHDP::Port*> > >::erase (this=0x7fffe4009920, __position=...) at /usr/include/c++/4.4/bits/stl_tree.h:1347
#2  0x00007ffff5fe312f in std::map<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID>, Sirikata::OHDP::Port*, std::less<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> >, std::allocator<std::pair<Sirikata::SST::EndPoint<Sirikata::OHDP::SpaceNodeID> const, Sirikata::OHDP::Port*> > >::erase (this=0x7fffe4009920, __position=...)
    at /usr/include/c++/4.4/bits/stl_map.h:567
#3  0x00007ffff5fe1d0f in Sirikata::SST::BaseDatagramLayer<Sirikata::OHDP::SpaceNodeID>::unlisten (this=0x7fffe4009910, ep=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/ohdp/SST.hpp:98
#4  0x00007ffff60052dc in Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::finalCleanup (this=0x7fffe4042870) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/network/SSTImpl.hpp:1136
#5  0x00007ffff5ffebbb in Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::~Connection (this=0x7fffe4042870, __in_chrg=<value optimized out>) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/network/SSTImpl.hpp:1243
#6  0x00007ffff6012aa2 in std::tr1::_Sp_deleter<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >::operator() (this=0x7fffe401b268, __p=0x7fffe4042870) at /usr/include/c++/4.4/tr1/shared_ptr.h:97
#7  0x00007ffff6011cdf in std::tr1::_Sp_counted_base_impl<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::_Sp_deleter<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >, (__gnu_cxx::_Lock_policy)2>::_M_dispose (this=0x7fffe401b250) at /usr/include/c++/4.4/tr1/shared_ptr.h:75
#8  0x00007ffff5f70952 in std::tr1::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release (this=0x7fffe401b250) at /usr/include/c++/4.4/tr1_impl/boost_sp_counted_base.h:140
#9  0x00007ffff5f6fbf1 in std::tr1::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count (this=0x9e7688, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/shared_ptr.h:153
#10 0x00007ffff5fe332c in std::tr1::__shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=0x9e7680, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/shared_ptr.h:358
#11 0x00007ffff5fe336c in std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >::~shared_ptr (this=0x9e7680, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/shared_ptr.h:834
#12 0x00007ffff5fe8174 in std::tr1::_Tuple_impl<1, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> > >::~_Tuple_impl (this=0x9e7680, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/tuple:70
#13 0x00007ffff5fe818e in std::tr1::_Tuple_impl<0, Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> > >::~_Tuple_impl (this=0x9e7680, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/tuple:70
#14 0x00007ffff5fe81a8 in std::tr1::tuple<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> > >::~tuple (this=0x9e7680, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1/tuple:129
#15 0x00007ffff5fe81c6 in std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::*)(std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)>(Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)>::~_Bind(void) (
    this=0x9e7670, __in_chrg=<value optimized out>) at /usr/include/c++/4.4/tr1_impl/functional:1115
#16 0x00007ffff5fff893 in _Base_manager<std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::*)(std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)>(Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)> >::_M_destroy(std::tr1::_Any_data &, std::tr1::false_type) (__victim=...) at /usr/include/c++/4.4/tr1_impl/functional:1523
#17 0x00007ffff5ff5370 in _Base_manager<std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::*)(std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)>(Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>*, std::tr1::shared_ptr<Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> >)> >::_M_manager(std::tr1::_Any_data &, const std::tr1::_Any_data &, std::tr1::_Manager_operation) (__dest=..., __source=..., __op=std::tr1::__destroy_functor) at /usr/include/c++/4.4/tr1_impl/functional:1547
#18 0x00007ffff66e545b in std::tr1::_Function_base::~_Function_base (this=Could not find the frame base for "std::tr1::_Function_base::~_Function_base()".
) at /usr/include/c++/4.4/tr1_impl/functional:1628
#19 0x00007ffff66e7f9a in std::tr1::function<void()>::~function(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1_impl/functional:1463
#20 0x00007ffff6810539 in boost::asio::detail::strand_service::handler_wrapper<std::tr1::function<void()> >::do_invoke(boost::asio::detail::strand_service::handler_base *, boost::asio::detail::strand_service &, boost::asio::detail::strand_service::strand_impl *&) (base=DWARF-2 expression error: DW_OP_reg operations must be used either alone or in conjuction with DW_OP_piece or DW_OP_bit_piece.
)
    at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/strand_service.hpp:261
#21 0x00007ffff680ec5f in boost::asio::detail::strand_service::handler_base::invoke (this=Unhandled dwarf expression opcode 0x18
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/strand_service.hpp:109
#22 0x00007ffff680ecce in boost::asio::detail::strand_service::invoke_current_handler::operator() (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/strand_service.hpp:144
#23 0x00007ffff68103e1 in boost::asio::asio_handler_invoke<boost::asio::detail::strand_service::invoke_current_handler> (function=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/handler_invoke_hook.hpp:62
#24 0x00007ffff68101cf in boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::strand_service::invoke_current_handler, boost::asio::detail::strand_service::invoke_current_handler> (function=Unhandled dwarf expression opcode 0x0
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/handler_invoke_helpers.hpp:41
#25 0x00007ffff680fd50 in handler_wrapper<boost::asio::detail::strand_service::invoke_current_handler>::do_call (base=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/handler_queue.hpp:192
#26 0x00007ffff67859b9 in boost::asio::detail::handler_queue::handler::invoke (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/handler_queue.hpp:39
#27 0x00007ffff678bfdf in boost::asio::detail::task_io_service<boost::asio::detail::epoll_reactor<false> >::do_one (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/task_io_service.hpp:268
#28 0x00007ffff67889f7 in boost::asio::detail::task_io_service<boost::asio::detail::epoll_reactor<false> >::run (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/task_io_service.hpp:103
#29 0x00007ffff6785ed9 in boost::asio::io_service::run (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/impl/io_service.ipp:68
#30 0x00007ffff68070b3 in Sirikata::Network::IOService::run (this=Unhandled dwarf expression opcode 0x0
) at /home/ewencp/sirikata.sirikata/libcore/src/network/IOService.cpp:118
#31 0x00007ffff6824ffa in Sirikata::Context::workerThread (this=Unhandled dwarf expression opcode 0x0
) at /home/ewencp/sirikata.sirikata/libcore/src/service/Context.cpp:117
#32 0x00007ffff682b2df in std::tr1::_Mem_fn<void (Sirikata::Context::*)()>::operator() (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1_impl/functional:552
#33 0x00007ffff682aaf4 in std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::Context::*)()>(Sirikata::Context*)>::__call<, 0>(const std::tr1::tuple<> &, std::tr1::_Index_tuple<0>) (this=Unhandled dwarf expression opcode 0x0
) at /usr/include/c++/4.4/tr1_impl/functional:1137
#34 0x00007ffff6829e49 in std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::Context::*)()>(Sirikata::Context*)>::operator()<>(void) (this=Could not find the frame base for "std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::Context::*)()>(Sirikata::Context*)>::operator()<>(void)".
) at /usr/include/c++/4.4/tr1_impl/functional:1191
#35 0x00007ffff6828ef7 in std::tr1::_Function_handler<void(), std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::Context::*)()>(Sirikata::Context*)> >::_M_invoke(const std::tr1::_Any_data &) (__functor=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1_impl/functional:1668
#36 0x00007ffff66e8ea9 in std::tr1::function<void()>::operator()(void) const (this=Unhandled dwarf expression opcode 0x0
) at /usr/include/c++/4.4/tr1_impl/functional:2024
#37 0x00007ffff66e82f8 in Sirikata::Thread::initThread(Sirikata::String, std::tr1::function<void()>, itimerval) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/util/Thread.hpp:131
#38 0x00007ffff6702b6f in std::tr1::_Mem_fn<void (Sirikata::Thread::*)(Sirikata::String, std::tr1::function<void()>, itimerval)>::operator()(Sirikata::Thread *, Sirikata::String, std::tr1::function<void()>, itimerval) const (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1_impl/functional:552
#39 0x00007ffff670295a in std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::Thread::*)(Sirikata::String, std::tr1::function<void()>, itimerval)>(Sirikata::Thread*, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::tr1::function<void()>, itimerval)>::__call<, 0, 1, 2, 3>(const std::tr1::tuple<> &, std::tr1::_Index_tuple<0, 1, 2, 3>)
    (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1_impl/functional:1137
#40 0x00007ffff6702839 in std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::Thread::*)(Sirikata::String, std::tr1::function<void()>, itimerval)>(Sirikata::Thread*, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::tr1::function<void()>, itimerval)>::operator()<>(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
) at /usr/include/c++/4.4/tr1_impl/functional:1191
#41 0x00007ffff67025fa in boost::detail::thread_data<std::tr1::_Bind<std::tr1::_Mem_fn<void (Sirikata::Thread::*)(Sirikata::String, std::tr1::function<void()>, itimerval)>(Sirikata::Thread*, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::tr1::function<void()>, itimerval)> >::run(void) (this=Asked for position 0 of stack, stack only has 0 elements on it.
)
    at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/thread/detail/thread.hpp:56
#42 0x00007ffff771233f in boost::(anonymous namespace)::thread_proxy (param=0x7fffe400a250) at libs/thread/src/pthread/thread.cpp:120
#43 0x00007ffff483a9ca in start_thread () from /lib/libpthread.so.0
#44 0x00007ffff4597cdd in clone () from /lib/libc.so.6
#45 0x0000000000000000 in ?? ()

The connections are different, but they share the same BaseDatagramLayer which seems to be getting corrupted. From the first thread:

#4  0x00007ffff5fe1d0f in Sirikata::SST::BaseDatagramLayer<Sirikata::OHDP::SpaceNodeID>::unlisten (this=0x7fffe4009910, ep=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/ohdp/SST.hpp:98
98          mAllocatedPorts.erase(it);
(gdb) p this
$4 = (Sirikata::SST::BaseDatagramLayer<Sirikata::OHDP::SpaceNodeID> * const) 0x7fffe4009910
(gdb) up
#5  0x00007ffff60052dc in Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::finalCleanup (this=0x7fffe4042340) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/network/SSTImpl.hpp:1136
1136         mDatagramLayer->unlisten(mLocalEndPoint);
(gdb) p this
$5 = (Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> * const) 0x7fffe4042340

and from the second thread:

#3  0x00007ffff5fe1d0f in Sirikata::SST::BaseDatagramLayer<Sirikata::OHDP::SpaceNodeID>::unlisten (this=0x7fffe4009910, ep=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/ohdp/SST.hpp:98
98          mAllocatedPorts.erase(it);
(gdb) p this
$6 = (Sirikata::SST::BaseDatagramLayer<Sirikata::OHDP::SpaceNodeID> * const) 0x7fffe4009910
(gdb) up
#4  0x00007ffff60052dc in Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID>::finalCleanup (this=0x7fffe4042870) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/network/SSTImpl.hpp:1136
1136         mDatagramLayer->unlisten(mLocalEndPoint);
(gdb) p this
$7 = (Sirikata::SST::Connection<Sirikata::OHDP::SpaceNodeID> * const) 0x7fffe4042870

I think this is just a matter of fixing the locking in the BaseDatagramLayers (both ODP and OHDP), and there's already some locking, but at least unlisten seems to need more.

ewencp commented 12 years ago

Likely fixed by 779a132b4c729 which makes sure a lock is acquired before the connection is cleaned up. We can reopen if it reappears.