azeey
commented
2 years ago I found an issue with pretty printed stack traces while I was running ign-gazebo. The stack trace only showed the bottom frame only
^CStack trace (most recent call last) in thread 2000:
#20 Object "", at 0xffffffffffffffff, inSo I've disabled it for now. It'll still show a useful backtrace, just not the code snippets. eg.
Stack trace (most recent call last) in thread 14356:
#20 Object "[0xffffffffffffffff]", at 0xffffffffffffffff, in
#19 Object "/lib/x86_64-linux-gnu/libc.so.6", at 0x7fd37420471e, in clone
#18 Object "/lib/x86_64-linux-gnu/libpthread.so.0", at 0x7fd373ecb6da, in
#17 Object "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", at 0x7fd371ab16de, in
#16 Object "/home/addisu/ws/citadel/install/lib/libignition-transport8.so.8", at 0x7fd36df0ea0f, in ignition::transport::v8::NodeShared::RunReceptionTask()
#15 Object "/home/addisu/ws/citadel/install/lib/libignition-transport8.so.8", at 0x7fd36df0b5b6, in ignition::transport::v8::NodeShared::RecvMsgUpdate()
#14 Object "/home/addisu/ws/citadel/install/lib/libignition-transport8.so.8", at 0x7fd36df06eec, in ignition::transport::v8::NodeShared::TriggerCallbacks(ignition::transport::v8::MessageInfo const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ignition::transport::v8::NodeShared::HandlerInfo const&)
#13 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-gui.so.3", at 0x7fd37052a952, in ignition::transport::v8::SubscriptionHandler<ignition::msgs::SerializedStepMap>::RunLocalCallback(google::protobuf::Message const&, ignition::transport::v8::MessageInfo const&)
#12 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-gui.so.3", at 0x7fd37052ff52, in std::function<void (ignition::msgs::SerializedStepMap const&, ignition::transport::v8::MessageInfo const&)>::operator()(ignition::msgs::SerializedStepMap const&, ignition::transport::v8::MessageInfo const&) const
#11 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-gui.so.3", at 0x7fd370517c2b, in std::_Function_handler<void (ignition::msgs::SerializedStepMap const&, ignition::transport::v8::MessageInfo const&), ignition::transport::v8::Node::Subscribe<ignition::gazebo::v3::GuiRunner, ignition::msgs::SerializedStepMap>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, void (ignition::gazebo::v3::GuiRunner::*)(ignition::msgs::SerializedStepMap const&), ignition::gazebo::v3::GuiRunner*, ignition::transport::v8::SubscribeOptions const&)::{lambda(ignition::msgs::SerializedStepMap const&, ignition::transport::v8::MessageInfo const&)#1}>::_M_invoke(std::_Any_data const&, ignition::msgs::SerializedStepMap const&, ignition::transport::v8::MessageInfo const&)
#10 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-gui.so.3", at 0x7fd3705115b6, in ignition::transport::v8::Node::Subscribe<ignition::gazebo::v3::GuiRunner, ignition::msgs::SerializedStepMap>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, void (ignition::gazebo::v3::GuiRunner::*)(ignition::msgs::SerializedStepMap const&), ignition::gazebo::v3::GuiRunner*, ignition::transport::v8::SubscribeOptions const&)::{lambda(ignition::msgs::SerializedStepMap const&, ignition::transport::v8::MessageInfo const&)#1}::operator()(ignition::msgs::SerializedStepMap const&, ignition::transport::v8::MessageInfo const&) const
#9 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-gui.so.3", at 0x7fd370513c25, in void std::_Bind<void (ignition::gazebo::v3::GuiRunner::*(ignition::gazebo::v3::GuiRunner*, std::_Placeholder<1>))(ignition::msgs::SerializedStepMap const&)>::operator()<ignition::msgs::SerializedStepMap const&, void>(ignition::msgs::SerializedStepMap const&)
#8 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-gui.so.3", at 0x7fd37051703d, in void std::_Bind<void (ignition::gazebo::v3::GuiRunner::*(ignition::gazebo::v3::GuiRunner*, std::_Placeholder<1>))(ignition::msgs::SerializedStepMap const&)>::__call<void, ignition::msgs::SerializedStepMap const&, 0ul, 1ul>(std::tuple<ignition::msgs::SerializedStepMap const&>&&, std::_Index_tuple<0ul, 1ul>)
#7 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-gui.so.3", at 0x7fd37051a256, in std::__invoke_result<void (ignition::gazebo::v3::GuiRunner::*&)(ignition::msgs::SerializedStepMap const&), ignition::gazebo::v3::GuiRunner*&, ignition::msgs::SerializedStepMap const&>::type std::__invoke<void (ignition::gazebo::v3::GuiRunner::*&)(ignition::msgs::SerializedStepMap const&), ignition::gazebo::v3::GuiRunner*&, ignition::msgs::SerializedStepMap const&>(void (ignition::gazebo::v3::GuiRunner::*&)(ignition::msgs::SerializedStepMap const&), ignition::gazebo::v3::GuiRunner*&, ignition::msgs::SerializedStepMap const&)
#6 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-gui.so.3", at 0x7fd37051c6dc, in void std::__invoke_impl<void, void (ignition::gazebo::v3::GuiRunner::*&)(ignition::msgs::SerializedStepMap const&), ignition::gazebo::v3::GuiRunner*&, ignition::msgs::SerializedStepMap const&>(std::__invoke_memfun_deref, void (ignition::gazebo::v3::GuiRunner::*&)(ignition::msgs::SerializedStepMap const&), ignition::gazebo::v3::GuiRunner*&, ignition::msgs::SerializedStepMap const&)
#5 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-gui.so.3", at 0x7fd3704ecadc, in ignition::gazebo::v3::GuiRunner::OnState(ignition::msgs::SerializedStepMap const&)
#4 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3.so.3", at 0x7fd36fa8a71f, in ignition::gazebo::v3::EntityComponentManager::ClearNewlyCreatedEntities()
#3 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-ign.so.3.9.0", at 0x7fd370b552b7, in std::lock_guard<std::mutex>::lock_guard(std::mutex&)
#2 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-ign.so.3.9.0", at 0x7fd370b5255b, in std::mutex::lock()
#1 Object "/home/addisu/ws/citadel/install/lib/libignition-gazebo3-ign.so.3.9.0", at 0x7fd370b4e4ec, in
#0 Object "/lib/x86_64-linux-gnu/libpthread.so.0", at 0x7fd373ecdfd0, in pthread_mutex_lock
chapulina
scpeters
🦟 Bug fix
Fixes #4
Summary
As described in #4, the backtrace currently generated includes a lot of information related to ruby that is not useful for diagnosing a crash. This PR vendors
Backwardand creates a small shared library that initializes it to handle the following signalsThis library gets loaded before
ignloads commands from other libraries so that it catches any signal generated by our C++ libraries. I believe @mjcarroll had added Backward to ign-gazebo before, but when we moved over from using standalone binaries to usingign, that functionality was lost.I added a
raise(SIGSEGV)somewhere in ign-gazebo to demonstrate:Before this PR
``` /home/addisu/ws/fortress/install/lib/ruby/ignition/cmdgazebo6.rb:431: [BUG] Segmentation fault at 0x000003e80000406b ruby 2.5.1p57 (2018-03-29 revision 63029) [x86_64-linux] -- Control frame information ----------------------------------------------- c:0007 p:---- s:0077 e:000076 CFUNC :call c:0006 p:0017 s:0054 e:000053 METHOD /home/addisu/ws/fortress/install/lib/ruby/ignition/cmdgazebo6.rb:431 c:0005 p:0166 s:0048 e:000047 BLOCK /home/addisu/ws/fortress/install/lib/ruby/ignition/cmdgazebo6.rb:460 [FINISH] c:0004 p:---- s:0045 e:000044 CFUNC :fork c:0003 p:0675 s:0041 e:000040 METHOD /home/addisu/ws/fortress/install/lib/ruby/ignition/cmdgazebo6.rb:456 c:0002 p:0772 s:0020 E:001540 EVAL /home/addisu/ws/fortress/install/bin/ign:298 [FINISH] c:0001 p:0000 s:0003 E:000730 (none) [FINISH] -- Ruby level backtrace information ---------------------------------------- /home/addisu/ws/fortress/install/bin/ign:298:in `With Backward (this PR)
``` Stack trace (most recent call last): #30 Object "", at 0xffffffffffffffff, in #29 Object "/home/addisu/.rbenv/versions/2.5.1/bin/ruby", at 0x5585b65bf919, in _start #28 Source "/build/glibc-S9d2JN/glibc-2.27/csu/../csu/libc-start.c", line 310, in __libc_start_main [0x7f2bf7746bf6] #27 Source "/tmp/ruby-build.20210125142253.8495.0c73M7/ruby-2.5.1/./main.c", line 42, in main [0x5585b65bf8ea] #26 Object "/home/addisu/.rbenv/versions/2.5.1/lib/libruby.so.2.5", at 0x7f2bf7bc506d, in ruby_run_node #25 Object "/home/addisu/.rbenv/versions/2.5.1/lib/libruby.so.2.5", at 0x7f2bf7bc25fc, in ruby_exec_node #24 Source "eval.c", line 0, in ruby_exec_internal [0x7f2bf7bc07a3] #23 Source "vm.c", line 0, in vm_exec [0x7f2bf7d21a9c] #22 Source "vm.c", line 0, in vm_exec_core [0x7f2bf7d1d2b2] #21 Source "vm.c", line 0, in vm_call_method [0x7f2bf7d23f9a] #20 Source "vm.c", line 0, in vm_call_method_each_type.part.135 [0x7f2bf7d2394b] #19 Source "vm.c", line 0, in vm_call_cfunc [0x7f2bf7d11c78] #18 Source "process.c", line 0, in rb_f_fork [0x7f2bf7c61191] #17 Object "/home/addisu/.rbenv/versions/2.5.1/lib/libruby.so.2.5", at 0x7f2bf7bc3640, in rb_protect #16 Object "/home/addisu/.rbenv/versions/2.5.1/lib/libruby.so.2.5", at 0x7f2bf7d2880a, in rb_yield #15 Source "vm.c", line 0, in invoke_iseq_block_from_c [0x7f2bf7d2266a] #14 Source "vm.c", line 0, in vm_exec [0x7f2bf7d21a9c] #13 Source "vm.c", line 0, in vm_exec_core [0x7f2bf7d1d2b2] #12 Source "vm.c", line 0, in vm_call_method [0x7f2bf7d23f9a] #11 Source "vm.c", line 0, in vm_call_method_each_type.part.135 [0x7f2bf7d2394b] #10 Source "vm.c", line 0, in vm_call_cfunc [0x7f2bf7d11c78] #9 Source "function.c", line 0, in function_call [0x7f2bf58018b4] #8 Object "/home/addisu/.rbenv/versions/2.5.1/lib/libruby.so.2.5", at 0x7f2bf7ce6ee7, in rb_thread_call_without_gvl #7 Source "function.c", line 0, in nogvl_ffi_call [0x7f2bf5801a77] #6 Object "/usr/lib/x86_64-linux-gnu/libffi.so.6", at 0x7f2bf55fa71e, in ffi_call #5 Object "/usr/lib/x86_64-linux-gnu/libffi.so.6", at 0x7f2bf55fadad, in ffi_call_unix64 #4 Source "/home/addisu/ws/fortress/src/ign-gazebo/src/ign.cc", line 354, in runServer [0x7f2bf3c5a907] 351: ignition::gazebo::Server server(serverConfig); 352: 353: // Run the server > 354: server.Run(true, _iterations, _run == 0); 355: 356: igndbg << "Shutting down ign-gazebo-server" << std::endl; 357: return 0; #3 Source "/home/addisu/ws/fortress/src/ign-gazebo/src/Server.cc", line 235, in ignition::gazebo::v6::Server::Run(bool, unsigned long, bool) [0x7f2bf34ea930] 232: } 233: 234: if (_blocking) > 235: return this->dataPtr->Run(_iterations); 236: 237: // Make sure two threads are not created 238: std::unique_lockChecklist
codecheckpassed (See contributing)Note to maintainers: Remember to use Squash-Merge