khuey
commented
5 years ago Can you attach gdb to rr itself and find out what value syscall_bp_vm has?
Open dothebart opened 5 years ago
khuey
commented
5 years ago Can you attach gdb to rr itself and find out what value syscall_bp_vm has?
dothebart
commented
5 years ago gdb> f 10
#10 0x0000000000770b49 in rr::ReplaySession::enter_syscall (this=0x255b3d0, t=0x2581070, constraints=...) at ../src/ReplaySession.cc:563
563 ASSERT(t, !syscall_bp_vm);
gdb> print syscall_bp_vm
$8 = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::AddressSpace*, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::AddressSpace*, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::AddressSpace> (use count 29, weak count 0) = {get() = 0x255cb70}and a bt full:
gdb> bt full
#0 0x00007f1a6232d704 in __GI___poll (fds=0x7fff5b7cd8e8, nfds=1, timeout=-1) at ../sysdeps/unix/sysv/linux/poll.c:29
resultvar = 18446744073709551100
sc_ret = <optimized out>
#1 0x00000000006709a8 in rr::poll_socket (sock_fd=..., events=1, timeoutMs=-1) at ../src/GdbConnection.cc:85
pfd = {fd = 186, events = 1, revents = 0}
ret = 0
#2 0x0000000000667932 in rr::poll_incoming (sock_fd=..., timeoutMs=-1) at ../src/GdbConnection.cc:93
No locals.
#3 0x00000000006677c0 in rr::GdbConnection::read_data_once (this=0x2cc4cd0) at ../src/GdbConnection.cc:107
nread = 46943440
buf = "h\r\000\000\000\000\000\000\002\000\000\000\000\000\000\000}և", '\000' <repeats 13 times>, "\001\000\000\000\000\000\000\000\202և\000\000\000\000\000\240??b\032\177\000\000\024\351(b\032\177\000\000\000\217;b\032\177\000\000\000\000\000\000\060\060\060\060\000\000\000\000\060\060\060\060\060\060\060\060", '\000' <repeats 12 times>, "\001\000\000\000\000\000\000\000\377\377\377\377\377\377\377\377\001\000\000\000\020", '\000' <repeats 15 times>, "\060\060\060\060\000\000\000\000\000\000\000\000\030\000\000\000\060\000\000\000\320\340|[\377\177\000\000\020\340|[\377\177\000\000\a\337|[\377\177\000\000\001\000\000\000\000\000\000\000\001\000\000\000\060\060\060\060\000\000\000\000\000\000\000\000"...
#4 0x0000000000668522 in rr::GdbConnection::read_packet (this=0x2cc4cd0) at ../src/GdbConnection.cc:298
checkedlen = 46943440
#5 0x000000000066e8d6 in rr::GdbConnection::get_request (this=0x2cc4cd0) at ../src/GdbConnection.cc:1302
No locals.
#6 0x00000000006826a4 in rr::GdbServer::process_debugger_requests (this=0x7fff5b7cf0a8, state=rr::GdbServer::REPORT_NORMAL) at ../src/GdbServer.cc:982
req = {type = rr::DREQ_GET_MEM, target = {pid = 21975, tid = 21979, static ANY = {pid = 0, tid = 0, static ANY = <same as static member of an already seen type>, static ALL = {pid = -1, tid = -1, static ANY = <same as static member of an already seen type>,
static ALL = <same as static member of an already seen type>}}, static ALL = <same as static member of an already seen type>}, suppress_debugger_stop = false, mem_ = {addr = 139626357136832, len = 64, data = std::vector of length 0, capacity 0}, watch_ = {addr = 140734728299088, kind = 6728169,
conditions = std::vector of length 0, capacity 0}, reg_ = {name = 46943440, {value = "\240:\357\002\000\000\000\000\340\341|[\377\177\000", value1 = 160 '\240', value2 = 15008, value4 = 49232544, value8 = 49232544}, size = 18446744069414584320, defined = 255}, restart_ = {param = 3,
param_str = <incomplete type>, type = (rr::RESTART_FROM_EVENT | rr::RESTART_FROM_CHECKPOINT | unknown: 6534280)}, cont_ = {run_direction = (unknown: 1534910944), actions = std::vector of length 0, capacity 0}, text_ = <incomplete type>, tls_ = {offset = 49232547, load_module = {ptr = 0}}, sym_ = {
has_address = 96, address = {ptr = 0}, name = <incomplete type>}, file_setfs_ = {pid = 1534929712}, file_open_ = {file_name = <incomplete type>, flags = 1, mode = 0}, file_pread_ = {fd = 51464384, size = 140734728301520, offset = 140734728298992}, file_close_ = {fd = 6327888}}
#7 0x0000000000687fcf in rr::GdbServer::emergency_debug (t=0x2581070) at ../src/GdbServer.cc:1685
features = {reverse_execution = false}
port = 17023
listen_fd = {fd = 185}
test_monitor_pid = 0x0
dbg = std::unique_ptr<rr::GdbConnection> = {get() = 0x0}
#8 0x00000000006b0e35 in rr::emergency_debug (t=0x2581070) at ../src/log.cc:392
sa = {__sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {__val = {0 <repeats 16 times>}}, sa_flags = 0, sa_restorer = 0x0}
record_session = 0x0
trace_stream = 0x255b4c0
#9 0x00000000006b0c52 in rr::EmergencyDebugOstream::~EmergencyDebugOstream (this=0x7fff5b7cf7b0) at ../src/log.cc:414
No locals.
#10 0x0000000000770b49 in rr::ReplaySession::enter_syscall (this=0x255b3d0, t=0x2581070, constraints=...) at ../src/ReplaySession.cc:563
syscall_instruction = {ptr = 1879048195}
#11 0x0000000000774ade in rr::ReplaySession::try_one_trace_step (this=0x255b3d0, t=0x2581070, constraints=...) at ../src/ReplaySession.cc:1316
No locals.
#12 0x0000000000775fcc in rr::ReplaySession::replay_step (this=0x255b3d0, constraints=...) at ../src/ReplaySession.cc:1562
t = 0x2581070
next_task = 0x2581070
result = {status = rr::REPLAY_CONTINUE, break_status = {task = 0x2581070, watchpoints_hit = std::vector of length 0, capacity 0, signal = std::unique_ptr<struct {...}> = {get() = 0x0}, breakpoint_hit = false, singlestep_complete = false, approaching_ticks_target = false, task_exit = false},
did_fast_forward = false}
#13 0x0000000000795931 in rr::ReplayTimeline::replay_step_forward (this=0x7fff5b7d2120, command=rr::RUN_CONTINUE, stop_at_time=15) at ../src/ReplayTimeline.cc:1382
before = {key = {trace_time = 578188, ticks = 492881, step_key = {action = rr::TSTEP_ENTER_SYSCALL}}, regs = {arch_ = rr::x86_64, u = {x86regs = {ebx = 16384, ecx = 0, edx = 1261117694, esi = 32509, edi = 1265311648, ebp = 32509, eax = 1279998336, xds = 32509, xes = 1261117440, xfs = 32509, xgs = 208,
orig_eax = 0, eip = 582, xcs = 0, eflags = 0, esp = 0, xss = 0}, x64regs = {r15 = 16384, r14 = 139626352943358, r13 = 139626357137312, r12 = 139626371824000, rbp = 139626352943104, rbx = 208, r11 = 582, r10 = 0, r9 = 0, r8 = 0, rax = 28, rcx = 18446744073709551615, rdx = 4, rsi = 12288,
rdi = 139626222313472, orig_rax = 18446744073709551615, rip = 1879048198, cs = 51, eflags = 582, rsp = 139626357136864, ss = 43, fs_base = 139626365527808, gs_base = 0, ds = 0, es = 0, fs = 0, gs = 0}}}, return_addresses = {addresses = {{ptr = 139626385132821}, {ptr = 0}, {ptr = 281474976710656}, {
ptr = 0}, {ptr = 0}, {ptr = 0}, {ptr = 0}, {ptr = 0}}}}
constraints = {command = rr::RUN_CONTINUE, stop_at_time = 15, ticks_target = 0, stop_before_states = std::vector of length 0, capacity 0}
did_hit_breakpoint = false
result = {status = rr::REPLAY_CONTINUE, break_status = {task = 0x0, watchpoints_hit = std::vector of length 0, capacity 0, signal = std::unique_ptr<struct {...}> = {get() = 0x0}, breakpoint_hit = false, singlestep_complete = false, approaching_ticks_target = false, task_exit = false},
did_fast_forward = false}
#14 0x00000000006847cc in rr::GdbServer::debug_one_step (this=0x7fff5b7d2058, last_resume_request=...) at ../src/GdbServer.cc:1178
signal_to_deliver = 0
command = rr::RUN_CONTINUE
result = {status = rr::REPLAY_CONTINUE, break_status = {task = 0x0, watchpoints_hit = std::vector of length 0, capacity 0, signal = std::unique_ptr<struct {...}> = {get() = 0x0}, breakpoint_hit = false, singlestep_complete = false, approaching_ticks_target = false, task_exit = false},
did_fast_forward = false}
req = {type = rr::DREQ_CONT, target = {pid = -1, tid = -1, static ANY = {pid = 0, tid = 0, static ANY = <same as static member of an already seen type>, static ALL = {pid = -1, tid = -1, static ANY = <same as static member of an already seen type>,
static ALL = <same as static member of an already seen type>}}, static ALL = <same as static member of an already seen type>}, suppress_debugger_stop = false, mem_ = {addr = 140734728298592, len = 39165904, data = std::vector of length 0, capacity 0}, watch_ = {addr = 39299408, kind = 39180784,
conditions = std::vector of length 0, capacity 0}, reg_ = {name = 1534910176, {value = "\025\064k\000\000\000\000\000P\251W\002\000\000\000", value1 = 21 '\025', value2 = 13333, value4 = 7025685, value8 = 7025685}, size = 10294912, defined = false}, restart_ = {param = 7025653,
param_str = <incomplete type>, type = rr::RESTART_FROM_PREVIOUS}, cont_ = {run_direction = rr::RUN_FORWARD, actions = std::vector of length 1, capacity 1 = {{type = rr::ACTION_CONTINUE, target = {pid = 21975, tid = -1, static ANY = {pid = 0, tid = 0,
static ANY = <same as static member of an already seen type>, static ALL = {pid = -1, tid = -1, static ANY = <same as static member of an already seen type>, static ALL = <same as static member of an already seen type>}}, static ALL = <same as static member of an already seen type>},
signal_to_deliver = 0}}}, text_ = <incomplete type>, tls_ = {offset = 7024729, load_module = {ptr = 0}}, sym_ = {has_address = 80, address = {ptr = 0}, name = <incomplete type>}, file_setfs_ = {pid = 7011548}, file_open_ = {file_name = <incomplete type>, flags = 39180629, mode = 0}, file_pread_ = {
fd = 39180610, size = 39262328, offset = 19}, file_close_ = {fd = 39180610}}
s = rr::GdbServer::CONTINUE_DEBUGGING
#15 0x0000000000685e57 in rr::GdbServer::serve_replay (this=0x7fff5b7d2058, flags=...) at ../src/GdbServer.cc:1552
last_resume_request = {type = rr::DREQ_CONT, target = {pid = -1, tid = -1, static ANY = {pid = 0, tid = 0, static ANY = <same as static member of an already seen type>, static ALL = {pid = -1, tid = -1, static ANY = <same as static member of an already seen type>,
static ALL = <same as static member of an already seen type>}}, static ALL = <same as static member of an already seen type>}, suppress_debugger_stop = false, mem_ = {addr = 140734728298592, len = 39165904, data = std::vector of length 0, capacity 0}, watch_ = {addr = 39299408, kind = 39180784,
conditions = std::vector of length 0, capacity 0}, reg_ = {name = 1534910176, {value = "\025\064k\000\000\000\000\000P\251W\002\000\000\000", value1 = 21 '\025', value2 = 13333, value4 = 7025685, value8 = 7025685}, size = 10294912, defined = false}, restart_ = {param = 7025653,
param_str = <incomplete type>, type = rr::RESTART_FROM_PREVIOUS}, cont_ = {run_direction = rr::RUN_FORWARD, actions = std::vector of length 1, capacity 1 = {{type = rr::ACTION_CONTINUE, target = {pid = 21975, tid = -1, static ANY = {pid = 0, tid = 0,
static ANY = <same as static member of an already seen type>, static ALL = {pid = -1, tid = -1, static ANY = <same as static member of an already seen type>, static ALL = <same as static member of an already seen type>}}, static ALL = <same as static member of an already seen type>},
signal_to_deliver = 0}}}, text_ = <incomplete type>, tls_ = {offset = 7024729, load_module = {ptr = 0}}, sym_ = {has_address = 80, address = {ptr = 0}, name = <incomplete type>}, file_setfs_ = {pid = 7011548}, file_open_ = {file_name = <incomplete type>, flags = 39180629, mode = 0}, file_pread_ = {
fd = 39180610, size = 39262328, offset = 19}, file_close_ = {fd = 39180610}}
port = 16463
probe = rr::PROBE_PORT
t = 0x255bce0
listen_fd = {fd = 13}
first_run_event = 15
#16 0x000000000076b58c in rr::replay (Python Exception <class 'gdb.error'> There is no member named _M_dataplus.:
trace_dir=, flags=...) at ../src/ReplayCommand.cc:452
conn_flags = {dbg_port = -1, dbg_host = <incomplete type>, keep_listening = false, debugger_params_write_pipe = 0x7fff5b7d23c8, debugger_name = <incomplete type>}
sa = {__sigaction_handler = {sa_handler = 0x76c3d0 <rr::handle_SIGINT_in_child(int)>, sa_sigaction = 0x76c3d0 <rr::handle_SIGINT_in_child(int)>}, sa_mask = {__val = {0 <repeats 16 times>}}, sa_flags = 268435456, sa_restorer = 0x0}
debugger_params_write_pipe = {fd = -1}
session = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::ReplaySession*, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::ReplaySession*, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplaySession> (use count 2, weak count 0) = {get() = 0x255b3d0}
server = {target = {pid = 21975, require_exec = false, event = 15}, dbg = std::unique_ptr<rr::GdbConnection> = {get() = 0x25716d0}, debuggee_tguid = {tid_ = 21975, serial_ = 1}, thread_db = std::unique_ptr<rr::ThreadDb> = {get() = 0x255dac0}, last_continue_tuid = {tid_ = 21975, serial_ = 1},
last_query_tuid = {tid_ = 21975, serial_ = 1}, final_event = 4294967295, stop_siginfo = {si_signo = 5, si_errno = 0, si_code = 0, __pad0 = 0, _sifields = {_pad = {0 <repeats 28 times>}, _kill = {si_pid = 0, si_uid = 0}, _timer = {si_tid = 0, si_overrun = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}},
_rt = {si_pid = 0, si_uid = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}}, _sigchld = {si_pid = 0, si_uid = 0, si_status = 0, si_utime = 0, si_stime = 0}, _sigfault = {si_addr = 0x0, si_addr_lsb = 0, _bounds = {_addr_bnd = {_lower = 0x0, _upper = 0x0}, _pkey = 0}}, _sigpoll = {si_band = 0,
si_fd = 0}, _sigsys = {_call_addr = 0x0, _syscall = 0, _arch = 0}}}, in_debuggee_end_state = false, stop_replaying_to_target = false, interrupt_pending = false, timeline = {session_flags = {redirect_stdio = true, share_private_mappings = false}, current = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::ReplaySession*, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::ReplaySession*, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplaySession> (use count 2, weak count 0) = {get() = 0x255b3d0}, current_at_or_after_mark = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 3, weak count 0) = {get() = 0x3112900}, marks = std::map with 62 elements = {[{trace_time = 14, ticks = 0, step_key = {
action = rr::TSTEP_RETIRE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x2560ae0}}, [{trace_time = 15, ticks = 0, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x25718d0}}, [{trace_time = 4689, ticks = 65753, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x257c480}}, [{trace_time = 11970, ticks = 10034, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2cd26e0}}, [{trace_time = 22296, ticks = 24448908, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2ef1890}}, [{trace_time = 36425, ticks = 39112, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2cbdb40}}, [{trace_time = 47425, ticks = 202628, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x2caf080}}, [{trace_time = 55017, ticks = 48937099, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2cceb50}}, [{trace_time = 61283, ticks = 72233683, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2cca110}}, [{trace_time = 67903, ticks = 31385, step_key = {action = rr::TSTEP_RETIRE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x322bc90}}, [{trace_time = 78749, ticks = 506475, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x2eeca90}}, [{trace_time = 88015, ticks = 1589677, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2efe620}}, [{trace_time = 97110, ticks = 113600450, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2cc52a0}}, [{trace_time = 107138, ticks = 127708909, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2ccb4f0}}, [{trace_time = 118223, ticks = 305033, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x3127580}}, [{trace_time = 129795, ticks = 3044342, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3128a40}}, [{trace_time = 141234, ticks = 2465001, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3111540}}, [{trace_time = 152566, ticks = 2508195, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x335c980}}, [{trace_time = 163936, ticks = 8664715, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x2ee2710}}, [{trace_time = 177931, ticks = 852517, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3128f20}}, [{trace_time = 192296, ticks = 666064, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2cc6eb0}}, [{trace_time = 206711, ticks = 749872, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2cce7a0}}, [{trace_time = 221094, ticks = 834162, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x3582210}}, [{trace_time = 235446, ticks = 1316277, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2cb9f80}}, [{trace_time = 249777, ticks = 7050937, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3128810}}, [{trace_time = 264202, ticks = 1087329, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2ee9810}}, [{trace_time = 278549, ticks = 1667287, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x2cc4960}}, [{trace_time = 292886, ticks = 865217, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2edb850}}, [{trace_time = 307224, ticks = 1981111, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2edc900}}, [{trace_time = 321576, ticks = 144979194, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2ccf850}}, [{trace_time = 334109, ticks = 2130933, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x2cc5440}}, [{trace_time = 345231, ticks = 12346775, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2cb92a0}}, [{trace_time = 354813, ticks = 2146887, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2ee54d0}}, [{trace_time = 362397, ticks = 179793170, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x358eec0}}, [{trace_time = 368396, ticks = 198797803, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x37b47c0}}, [{trace_time = 374062, ticks = 2971688, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x35912d0}}, [{trace_time = 381783, ticks = 259662189, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3593d40}}, [{trace_time = 390891, ticks = 2320353, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2ed9110}}, [{trace_time = 397307, ticks = 44654407, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x2ed80b0}}, [{trace_time = 404333, ticks = 341055240, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3361970}}, [{trace_time = 411736, ticks = 15518204, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3345580}}, [{trace_time = 418915, ticks = 2471387, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3345ca0}}, [{trace_time = 424861, ticks = 403476112, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x2ef2650}}, [{trace_time = 430811, ticks = 426793091, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3110b20}}, [{trace_time = 437577, ticks = 609458, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x37ca9f0}}, [{trace_time = 444287, ticks = 1697034, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2cac0a0}}, [{trace_time = 453182, ticks = 12607498, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x2ee2d00}}, [{trace_time = 460483, ticks = 673987, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2ccd070}}, [{trace_time = 467740, ticks = 536287076, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3126970}}, [{trace_time = 474869, ticks = 18020930, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x2edc010}}, [{trace_time = 483605, ticks = 2786211, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x358b190}}, [{trace_time = 491221, ticks = 595291034, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2ccced0}}, [{trace_time = 498287, ticks = 614804811, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x2eebb70}}, [{trace_time = 505671, ticks = 631726668, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3574790}}, [{trace_time = 513085, ticks = 646106537, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x31267c0}}, [{trace_time = 521094, ticks = 6374597, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3104e80}}, [{trace_time = 529324, ticks = 681148553, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {get() = 0x3345110}}, [{trace_time = 537656, ticks = 6767966, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x2cbfea0}}, [{trace_time = 545234, ticks = 1516699, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 1, weak count 0) = {
get() = 0x37c8cb0}}, [{trace_time = 551368, ticks = 755906211, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x3593ee0}}, [{trace_time = 560663, ticks = 2664232, step_key = {
action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x37c9bc0}}, [{trace_time = 571654, ticks = 9234246, step_key = {action = rr::TSTEP_NONE}}] = std::vector of length 1, capacity 1 = {warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 3, weak count 0) = {get() = 0x3112900}}}, marks_with_checkpoints = std::map with 8 elements = {[{trace_time = 14, ticks = 0, step_key = {action = rr::TSTEP_RETIRE}}] = 1, [{trace_time = 15, ticks = 0, step_key = {action = rr::TSTEP_NONE}}] = 1, [{
trace_time = 362397, ticks = 179793170, step_key = {action = rr::TSTEP_NONE}}] = 1, [{trace_time = 474869, ticks = 18020930, step_key = {action = rr::TSTEP_NONE}}] = 1, [{trace_time = 537656, ticks = 6767966, step_key = {action = rr::TSTEP_NONE}}] = 1, [{trace_time = 551368, ticks = 755906211,
step_key = {action = rr::TSTEP_NONE}}] = 1, [{trace_time = 560663, ticks = 2664232, step_key = {action = rr::TSTEP_NONE}}] = 1, [{trace_time = 571654, ticks = 9234246, step_key = {action = rr::TSTEP_NONE}}] = 1}, breakpoints = std::set with 2 elements = {[0] = std::tuple containing = {
[1] = {<rr::TaskishUid<rr::AddressSpace>> = {tid_ = 21975, serial_ = 1}, exec_count_ = 1}, [2] = {ptr = 94192278195648}, [3] = std::unique_ptr<rr::BreakpointCondition> = {get() = 0x0}}, [1] = std::tuple containing = {[1] = {<rr::TaskishUid<rr::AddressSpace>> = {tid_ = 21975, serial_ = 1},
exec_count_ = 1}, [2] = {ptr = 139626385376896}, [3] = std::unique_ptr<rr::BreakpointCondition> = {get() = 0x0}}}, watchpoints = std::set with 0 elements, breakpoints_applied = true, reverse_execution_barrier_event = 15, reverse_exec_checkpoints = std::map with 7 elements = {[{ptr = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x2560ae0}}] = 34371, [{ptr = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x358eec0}}] = 16101169, [{ptr = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x2edc010}}] = 24144819, [{ptr = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x2cbfea0}}] = 28187197, [{ptr = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x3593ee0}}] = 29189245, [{ptr = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x37c9bc0}}] = 29689805, [{ptr = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 3, weak count 0) = {get() = 0x3112900}}] = 30189906}, no_watchpoints_hit_interval_start = {ptr = std::shared_ptr<rr::ReplayTimeline::InternalMark> (empty) = {get() = 0x0}}, no_watchpoints_hit_interval_end = {
ptr = std::shared_ptr<rr::ReplayTimeline::InternalMark> (empty) = {get() = 0x0}}, reverse_exec_short_checkpoint = {ptr = std::shared_ptr<rr::ReplayTimeline::InternalMark> (empty) = {get() = 0x0}}}, emergency_debug_session = 0x0, debugger_restart_checkpoint = {mark = {ptr = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr_inplace<rr::ReplayTimeline::InternalMark, std::allocator<rr::ReplayTimeline::InternalMark>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ReplayTimeline::InternalMark> (use count 2, weak count 0) = {get() = 0x25718d0}}, last_continue_tuid = {tid_ = 21975, serial_ = 1}, is_explicit = rr::GdbServer::Checkpoint::EXPLICIT, where = <incomplete type>}, checkpoints = std::map with 0 elements, symbols = std::set with 67 elements = {
[0] = <incomplete type>, [1] = <incomplete type>, [2] = <incomplete type>, [3] = <incomplete type>, [4] = <incomplete type>, [5] = <incomplete type>, [6] = <incomplete type>, [7] = <incomplete type>, [8] = <incomplete type>, [9] = <incomplete type>, [10] = <incomplete type>, [11] = <incomplete type>,
[12] = <incomplete type>, [13] = <incomplete type>, [14] = <incomplete type>, [15] = <incomplete type>, [16] = <incomplete type>, [17] = <incomplete type>, [18] = <incomplete type>, [19] = <incomplete type>, [20] = <incomplete type>, [21] = <incomplete type>, [22] = <incomplete type>,
[23] = <incomplete type>, [24] = <incomplete type>, [25] = <incomplete type>, [26] = <incomplete type>, [27] = <incomplete type>, [28] = <incomplete type>, [29] = <incomplete type>, [30] = <incomplete type>, [31] = <incomplete type>, [32] = <incomplete type>, [33] = <incomplete type>,
[34] = <incomplete type>, [35] = <incomplete type>, [36] = <incomplete type>, [37] = <incomplete type>, [38] = <incomplete type>, [39] = <incomplete type>, [40] = <incomplete type>, [41] = <incomplete type>, [42] = <incomplete type>, [43] = <incomplete type>, [44] = <incomplete type>,
[45] = <incomplete type>, [46] = <incomplete type>, [47] = <incomplete type>, [48] = <incomplete type>, [49] = <incomplete type>, [50] = <incomplete type>, [51] = <incomplete type>, [52] = <incomplete type>, [53] = <incomplete type>, [54] = <incomplete type>, [55] = <incomplete type>,
[56] = <incomplete type>, [57] = <incomplete type>, [58] = <incomplete type>, [59] = <incomplete type>, [60] = <incomplete type>, [61] = <incomplete type>, [62] = <incomplete type>, [63] = <incomplete type>, [64] = <incomplete type>, [65] = <incomplete type>, [66] = <incomplete type>}, Python Exception <class 'gdb.error'> No type named std::_Rb_tree_node<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >>.:
symbols_iter = {
_M_node = 0x7fff5b7d22f8}, files = std::map with 0 elements, file_scope_pid = 0}
target = {pid = 0, require_exec = false, event = 0}
debugger_params_pipe = {3, 4}
sa = {__sigaction_handler = {sa_handler = 0x7f1a62c2a410, sa_sigaction = 0x7f1a62c2a410}, sa_mask = {__val = {139751296358630, 5952209431, 140734728314688, 140734728314704, 139751302801256, 0, 1, 139751302800400, 4294967295, 4294967296, 139751296201664, 139751302800400, 139751302801256, 0, 0,
140734728316144}}, sa_flags = -1, sa_restorer = 0x7f1a62c7c8c3 <_dl_fixup+211>}
#17 0x000000000076a2a0 in rr::ReplayCommand::run (this=0x9d2a18 <rr::ReplayCommand::singleton>, args=warning: Type size unknown, assuming 1. Try casting to a known type, or void *.
warning: Type size unknown, assuming 1. Try casting to a known type, or void *.
std::vector of length 0, capacity 64) at ../src/ReplayCommand.cc:567
found_dir = true
trace_dir = <incomplete type>
flags = {goto_event = 0, singlestep_to_event = 0, target_process = 0, target_command = <incomplete type>, process_created_how = rr::ReplayFlags::CREATED_NONE, dont_launch_debugger = false, dbg_port = -1, dbg_host = <incomplete type>, keep_listening = false, gdb_options = warning: Type size unknown, assuming 1. Try casting to a known type, or void *.
warning: Type size unknown, assuming 1. Try casting to a known type, or void *.
std::vector of length 0, capacity 0,
gdb_binary_file_path = <incomplete type>, redirect = true, share_private_mappings = false}
#18 0x0000000000825340 in main (argc=3, argv=0x7fff5b7d2b38) at ../src/main.cc:251
args = warning: Type size unknown, assuming 1. Try casting to a known type, or void *.
warning: Type size unknown, assuming 1. Try casting to a known type, or void *.
std::vector of length 0, capacity 64
command = 0x9d2a18 <rr::ReplayCommand::singleton>#12 0x0000000000775fcc in rr::ReplaySession::replay_step (this=0x255b3d0, constraints=...) at ../src/ReplaySession.cc:1562
1562 if (try_one_trace_step(t, constraints) == INCOMPLETE) {
gdb> print *this
$9 = {<rr::Session> = {_vptr$Session = 0x895440 <vtable for rr::ReplaySession+16>, vm_map = std::map with 1 element = {[{<rr::TaskishUid<rr::AddressSpace>> = {tid_ = 21975, serial_ = 1}, exec_count_ = 1}] = 0x255cb70}, task_map = std::map with 28 elements = {[11866] = 0x3573510, [13766] = 0x33393c0,
[21975] = 0x255bce0, [21979] = 0x2581070, [21985] = 0x2589520, [22561] = 0x258a820, [22564] = 0x2a97230, [22612] = 0x2a98d70, [22617] = 0x2a998b0, [22638] = 0x2a9a410, [22639] = 0x2a9aaf0, [22642] = 0x2a9b230, [22738] = 0x2a9f350, [22740] = 0x2a9e350, [22742] = 0x2a9d790, [22744] = 0x2a9cbd0,
[22747] = 0x2a9c190, [22748] = 0x2a9d240, [22749] = 0x2a9ff10, [22750] = 0x2aa0850, [22752] = 0x2aa11d0, [22755] = 0x2aa1f20, [22760] = 0x2aa2a10, [22762] = 0x2aa3390, [22770] = 0x2aa4380, [22771] = 0x2aa4d40, [22772] = 0x2aa5a80, [27349] = 0x2edec70}, thread_group_map = std::map with 1 element = {[{
tid_ = 21975, serial_ = 1}] = 0x255c030}, clone_completion = std::unique_ptr<rr::Session::CloneCompletion> = {get() = 0x0}, statistics_ = {bytes_written = 313401874, ticks_processed = 937543692, syscalls_performed = 703785}, tracee_socket = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::ScopedFd*, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::ScopedFd*, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ScopedFd> (use count 9, weak count 0) = {
get() = 0x2559eb0}, tracee_socket_fd_number = 1001, next_task_serial_ = 72, spawned_task_error_fd_ = {fd = -1}, syscall_seccomp_ordering_ = rr::Session::PTRACE_SYSCALL_BEFORE_SECCOMP_UNKNOWN, ticks_semantics_ = rr::TICKS_RETIRED_CONDITIONAL_BRANCHES, done_initial_exec_ = true, visible_execution_ = true},
emu_fs = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::EmuFs*, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::EmuFs*, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::EmuFs> (use count 1, weak count 0) = {get() = 0x255b960}, trace_in = {<rr::TraceStream> = {trace_dir = <incomplete type>, bind_to_cpu = 7, ticks_semantics_ = rr::TICKS_RETIRED_CONDITIONAL_BRANCHES, global_time = 578188}, xcr0_ = 7, readers = {std::unique_ptr<rr::CompressedReader> = {
get() = 0x255a8b0}, std::unique_ptr<rr::CompressedReader> = {get() = 0x255ba60}, std::unique_ptr<rr::CompressedReader> = {get() = 0x255bb20}, std::unique_ptr<rr::CompressedReader> = {get() = 0x255bbe0}}, cpuid_records_ = std::vector of length 92, capacity 92 = {{eax_in = 0, ecx_in = 4294967295, out = {
eax = 13, ebx = 1970169159, ecx = 1818588270, edx = 1231384169}}, {eax_in = 1, ecx_in = 4294967295, out = {eax = 132823, ebx = 18876416, ecx = 532603839, edx = 3219913727}}, {eax_in = 2, ecx_in = 4294967295, out = {eax = 1979931137, ebx = 15774463, ecx = 0, edx = 13238272}}, {eax_in = 3,
ecx_in = 4294967295, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 4, ecx_in = 0, out = {eax = 1006649633, ebx = 29360191, ecx = 63, edx = 0}}, {eax_in = 4, ecx_in = 1, out = {eax = 1006649634, ebx = 29360191, ecx = 63, edx = 0}}, {eax_in = 4, ecx_in = 2, out = {eax = 1006649667, ebx = 29360191,
ecx = 511, edx = 0}}, {eax_in = 4, ecx_in = 3, out = {eax = 1007141219, ebx = 79691839, ecx = 8191, edx = 6}}, {eax_in = 4, ecx_in = 4, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 5, ecx_in = 4294967295, out = {eax = 64, ebx = 64, ecx = 3, edx = 135456}}, {eax_in = 6, ecx_in = 4294967295, out = {
eax = 119, ebx = 2, ecx = 9, edx = 0}}, {eax_in = 7, ecx_in = 0, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 8, ecx_in = 4294967295, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 9, ecx_in = 4294967295, out = {eax = 1, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 10, ecx_in = 4294967295,
out = {eax = 120587267, ebx = 0, ecx = 0, edx = 1539}}, {eax_in = 11, ecx_in = 0, out = {eax = 1, ebx = 2, ecx = 256, edx = 1}}, {eax_in = 11, ecx_in = 1, out = {eax = 5, ebx = 8, ecx = 513, edx = 1}}, {eax_in = 11, ecx_in = 2, out = {eax = 0, ebx = 0, ecx = 2, edx = 1}}, {eax_in = 12, ecx_in = 4294967295,
out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 0, out = {eax = 7, ebx = 832, ecx = 832, edx = 0}}, {eax_in = 13, ecx_in = 1, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 2, out = {eax = 256, ebx = 576, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 3, out = {eax = 0,
ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 4, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 5, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 6, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 7, out = {eax = 0, ebx = 0, ecx = 0,
edx = 0}}, {eax_in = 13, ecx_in = 8, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 9, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 10, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 11, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {
eax_in = 13, ecx_in = 12, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 13, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 14, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 15, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13,
ecx_in = 16, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 17, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 18, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 19, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 20, out = {
eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 21, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 22, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 23, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 24, out = {eax = 0, ebx = 0,
ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 25, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 26, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 27, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 28, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}},
{eax_in = 13, ecx_in = 29, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 30, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 31, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 32, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13,
ecx_in = 33, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 34, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 35, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 36, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 37, out = {
eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 38, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 39, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 40, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 41, out = {eax = 0, ebx = 0,
ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 42, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 43, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 44, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 45, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}},
{eax_in = 13, ecx_in = 46, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 47, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 48, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 49, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13,
ecx_in = 50, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 51, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 52, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 53, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 54, out = {
eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 55, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 56, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 57, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 58, out = {eax = 0, ebx = 0,
ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 59, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 60, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 61, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 13, ecx_in = 62, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}},
{eax_in = 13, ecx_in = 63, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 2147483648, ecx_in = 4294967295, out = {eax = 2147483656, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 2147483649, ecx_in = 4294967295, out = {eax = 0, ebx = 0, ecx = 1, edx = 739248128}}, {eax_in = 2147483650, ecx_in = 4294967295,
out = {eax = 538976288, ebx = 538976288, ecx = 1702129225, edx = 693250156}}, {eax_in = 2147483651, ecx_in = 4294967295, out = {eax = 1919894304, ebx = 1297360997, ecx = 929636393, edx = 842543917}}, {eax_in = 2147483652, ecx_in = 4294967295, out = {eax = 1346576432, ebx = 541073493, ecx = 808857139,
edx = 8013895}}, {eax_in = 2147483653, ecx_in = 4294967295, out = {eax = 0, ebx = 0, ecx = 0, edx = 0}}, {eax_in = 2147483654, ecx_in = 4294967295, out = {eax = 0, ebx = 0, ecx = 16801856, edx = 0}}, {eax_in = 2147483655, ecx_in = 4294967295, out = {eax = 0, ebx = 0, ecx = 0, edx = 256}}, {
eax_in = 2147483656, ecx_in = 4294967295, out = {eax = 12334, ebx = 0, ecx = 0, edx = 0}}}, raw_recs = std::vector of length 0, capacity 16, trace_uses_cpuid_faulting = false}, trace_frame = {global_time = 578188, tid_ = 21979, ev = {event_type = rr::EV_SYSCALL, {desched = {rec = {ptr = 1}}, signal = {
siginfo = {si_signo = 1, si_errno = 0, si_code = 1, __pad0 = 0, _sifields = {_pad = {0 <repeats 28 times>}, _kill = {si_pid = 0, si_uid = 0}, _timer = {si_tid = 0, si_overrun = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}}, _rt = {si_pid = 0, si_uid = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}},
_sigchld = {si_pid = 0, si_uid = 0, si_status = 0, si_utime = 0, si_stime = 0}, _sigfault = {si_addr = 0x0, si_addr_lsb = 0, _bounds = {_addr_bnd = {_lower = 0x0, _upper = 0x0}, _pkey = 0}}, _sigpoll = {si_band = 0, si_fd = 0}, _sigsys = {_call_addr = 0x0, _syscall = 0, _arch = 0}}},
deterministic = rr::NONDETERMINISTIC_SIG, disposition = rr::DISPOSITION_FATAL}, syscall = {arch_ = rr::x86_64, regs = {arch_ = rr::x86_64, u = {x86regs = {ebx = 0, ecx = 0, edx = 0, esi = 0, edi = 0, ebp = 0, eax = 0, xds = 0, xes = 0, xfs = 0, xgs = 0, orig_eax = 0, eip = 0, xcs = 0, eflags = 0, esp = 0,
xss = 0}, x64regs = {r15 = 0, r14 = 0, r13 = 0, r12 = 0, rbp = 0, rbx = 0, r11 = 0, r10 = 0, r9 = 0, r8 = 0, rax = 0, rcx = 0, rdx = 0, rsi = 0, rdi = 0, orig_rax = 0, rip = 0, cs = 0, eflags = 0, rsp = 0, ss = 0, fs_base = 0, gs_base = 0, ds = 0, es = 0, fs = 0, gs = 0}}}, desched_rec = {ptr = 0},
write_offset = -1, exec_fds_to_close = std::vector of length 0, capacity 0, opened = std::vector of length 0, capacity 0, state = rr::ENTERING_SYSCALL, number = 1, switchable = rr::PREVENT_SWITCH, is_restart = false, failed_during_preparation = false, in_sysemu = false}, syscallbuf_flush = {
mprotect_records = std::vector of length 0, capacity 0}}}, ticks_ = 492882, monotonic_time_ = 4261049.0819752729, recorded_regs = {arch_ = rr::x86_64, u = {x86regs = {ebx = 1261117470, ecx = 32509, edx = 0, esi = 0, edi = 1273692536, ebp = 32509, eax = 1279998336, xds = 32509, xes = 1273692368,
xfs = 32509, xgs = 1261117440, orig_eax = 32509, eip = 582, xcs = 0, eflags = 0, esp = 0, xss = 0}, x64regs = {r15 = 139626352943134, r14 = 0, r13 = 139626365518200, r12 = 139626371824000, rbp = 139626365518032, rbx = 139626352943104, r11 = 582, r10 = 0, r9 = 0, r8 = 0, rax = 18446744073709551578,
rcx = 18446744073709551615, rdx = 79, rsi = 139626385266469, rdi = 2, orig_rax = 1, rip = 1879048197, cs = 51, eflags = 582, rsp = 139626357137024, ss = 43, fs_base = 139626365527808, gs_base = 0, ds = 0, es = 0, fs = 0, gs = 0}}}, recorded_extra_regs = {format_ = rr::ExtraRegisters::NONE,
arch_ = rr::x86_64, data_ = std::vector of length 0, capacity 0}}, current_step = {action = rr::TSTEP_ENTER_SYSCALL, {syscall = {arch = rr::x86, number = 1}, target = {ticks = 4294967296, signo = 0}, flush = {stop_breakpoint_addr = 4294967296}}}, ticks_at_start_of_event = 492881, cpuid_bug_detector = {
trace_rcb_count_at_last_geteuid32 = 27, actual_rcb_count_at_last_geteuid32 = 304, detected_cpuid_bug = false}, last_siginfo_ = {si_signo = 0, si_errno = 0, si_code = 0, __pad0 = 0, _sifields = {_pad = {0 <repeats 28 times>}, _kill = {si_pid = 0, si_uid = 0}, _timer = {si_tid = 0, si_overrun = 0, si_sigval = {
sival_int = 0, sival_ptr = 0x0}}, _rt = {si_pid = 0, si_uid = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}}, _sigchld = {si_pid = 0, si_uid = 0, si_status = 0, si_utime = 0, si_stime = 0}, _sigfault = {si_addr = 0x0, si_addr_lsb = 0, _bounds = {_addr_bnd = {_lower = 0x0, _upper = 0x0}, _pkey = 0}},
_sigpoll = {si_band = 0, si_fd = 0}, _sigsys = {_call_addr = 0x0, _syscall = 0, _arch = 0}}}, flags = {redirect_stdio = true, share_private_mappings = false}, did_fast_forward = false, trace_start_time = 4260298.9408207247, syscall_bp_vm = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::AddressSpace*, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::AddressSpace*, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::AddressSpace> (use count 29, weak count 0) = {
get() = 0x255cb70}, syscall_bp_addr = {ptr = 1879048195}}hm, is the debian-gdb to old already? GNU gdb (Debian 8.1-4+b1) 8.1
khuey
commented
5 years ago Ok, so the tracee is at 70000008 (the untraced replay-only syscall instruction) but syscall_bp_addr is 70000003 (the privileged traced syscall instruction). That makes sense, since the write(2) we're replaying has a recorded RIP of 70000005. A privileged traced write had to come from syscallbuf's logmsg, which would only be triggered if we fatally asserted during recording.
If you rr dump 578188-578200 what does that output? Do you have the stderr from recording this trace? Was there a message about a fatal assertion?
rocallahan
commented
5 years ago You could replay the execution, when it fails run the emergency-debugger instructions, and then in the emergency debugger print (char*)0x7efd4d185725. That should tell us what failed during recording.
dothebart
commented
5 years ago @khuey sorry, I don't have the original output during the recording session anymore, just whats printed during the replay. I was hunting a race in arangodb where I was running unittests in loop, and loading the machine with a parallel make -j 20. The remaining processes of the cluster flodded the output, so the actual incident was out of sight.
gdb> print (char*)0x7efd4d185725
$1 = 0x7efd4d185725 "../src/preload/syscallbuf.c:907: Fatal error: \"Record syscall number mismatch\"\n"The dump is pretty long, should I attach it? Heres the last dozend lines:
rax:0x1c rbx:0xd0 rcx:0xffffffffffffffff rdx:0x4 rsi:0x3000 rdi:0x7efd4361e000 rbp:0x7efd4b2b2000 rsp:0x7efd4b6b1de0 r8:0x0 r9:0x0 r10:0x0 r11:0x246 r12:0x7efd4c4b3980 r13:0x7efd4b6b1fa0 r14:0x7efd4b2b20fe r15:0x4000 rip:0x70000006 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0xffffffffffffff09 f
s_base:0x7efd4beb2700 gs_base:0x0
}
{
real_time:4261049.080697 global_time:578184, event:`SYSCALLBUF_RESET' tid:21979, ticks:492881
}
{
real_time:4261049.081679 global_time:578185, event:`SYSCALLBUF_FLUSH' tid:22772, ticks:785709062
}
{
real_time:4261049.081705 global_time:578186, event:`SYSCALL: futex' (state:ENTERING_SYSCALL) tid:22772, ticks:785709062
rax:0xffffffffffffffda rbx:0xca rcx:0xffffffffffffffff rdx:0x0 rsi:0x80 rdi:0x7efd42b00110 rbp:0x0 rsp:0x7efd2e0f5de0 r8:0x0 r9:0x43d r10:0x0 r11:0x246 r12:0xc44 r13:0x7efd2e0f5fa0 r14:0x0 r15:0x7efd42b00110 rip:0x70000002 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0xca fs_base:0x7efd2e8f6700 g
s_base:0x0
}
{
real_time:4261049.081722 global_time:578187, event:`SYSCALLBUF_RESET' tid:22772, ticks:785709062
}
{
real_time:4261049.081975 global_time:578188, event:`SYSCALL: write' (state:ENTERING_SYSCALL) tid:21979, ticks:492882
rax:0xffffffffffffffda rbx:0x7efd4b2b2000 rcx:0xffffffffffffffff rdx:0x4f rsi:0x7efd4d185725 rdi:0x2 rbp:0x7efd4beb00d0 rsp:0x7efd4b6b1e80 r8:0x0 r9:0x0 r10:0x0 r11:0x246 r12:0x7efd4c4b3980 r13:0x7efd4beb0178 r14:0x0 r15:0x7efd4b2b201e rip:0x70000005 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0x1 fs_base:0x7efd4beb2700 gs_base:0x0
}
{
real_time:4261049.082094 global_time:578189, event:`SYSCALL: write' (state:EXITING_SYSCALL) tid:21979, ticks:492882
rax:0x4f rbx:0x7efd4b2b2000 rcx:0xffffffffffffffff rdx:0x4f rsi:0x7efd4d185725 rdi:0x2 rbp:0x7efd4beb00d0 rsp:0x7efd4b6b1e80 r8:0x0 r9:0x0 r10:0x0 r11:0x246 r12:0x7efd4c4b3980 r13:0x7efd4beb0178 r14:0x0 r15:0x7efd4b2b201e rip:0x70000005 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0x1 fs_base:0x7efd4beb2700 gs_base:0x0
}
{
real_time:4261049.082376 global_time:578190, event:`SYSCALL: getpid' (state:ENTERING_SYSCALL) tid:21979, ticks:492882
rax:0xffffffffffffffda rbx:0x7efd4b2b2000 rcx:0xffffffffffffffff rdx:0x0 rsi:0x0 rdi:0x0 rbp:0x7efd4beb00d0 rsp:0x7efd4b6b1e80 r8:0x0 r9:0x0 r10:0x0 r11:0x246 r12:0x7efd4c4b3980 r13:0x7efd4beb0178 r14:0x0 r15:0x7efd4b2b201e rip:0x70000005 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0x27 fs_base:0x7efd4beb2700 gs_base:0x0
}
{
real_time:4261049.082455 global_time:578191, event:`SYSCALL: getpid' (state:EXITING_SYSCALL) tid:21979, ticks:492882
rax:0x55d7 rbx:0x7efd4b2b2000 rcx:0xffffffffffffffff rdx:0x0 rsi:0x0 rdi:0x0 rbp:0x7efd4beb00d0 rsp:0x7efd4b6b1e80 r8:0x0 r9:0x0 r10:0x0 r11:0x246 r12:0x7efd4c4b3980 r13:0x7efd4beb0178 r14:0x0 r15:0x7efd4b2b201e rip:0x70000005 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0x27 fs_base:0x7efd4beb2700 gs_base:0x0
}
{
real_time:4261049.082734 global_time:578192, event:`SYSCALL: kill' (state:ENTERING_SYSCALL) tid:21979, ticks:492882
rax:0xffffffffffffffda rbx:0x7efd4b2b2000 rcx:0xffffffffffffffff rdx:0x0 rsi:0x6 rdi:0x55d7 rbp:0x7efd4beb00d0 rsp:0x7efd4b6b1e80 r8:0x0 r9:0x0 r10:0x0 r11:0x246 r12:0x7efd4c4b3980 r13:0x7efd4beb0178 r14:0x0 r15:0x7efd4b2b201e rip:0x70000005 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0x3e fs_base:0x7efd4beb2700 gs_base:0x0
}
{
real_time:4261049.082824 global_time:578193, event:`SYSCALL: kill' (state:EXITING_SYSCALL) tid:21979, ticks:492882
rax:0x0 rbx:0x7efd4b2b2000 rcx:0xffffffffffffffff rdx:0x0 rsi:0x6 rdi:0x55d7 rbp:0x7efd4beb00d0 rsp:0x7efd4b6b1e80 r8:0x0 r9:0x0 r10:0x0 r11:0x246 r12:0x7efd4c4b3980 r13:0x7efd4beb0178 r14:0x0 r15:0x7efd4b2b201e rip:0x70000005 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0x3e fs_base:0x7efd4beb2700 gs_base:0x0
}
{
real_time:4261049.083412 global_time:578194, event:`SYSCALLBUF_FLUSH' tid:21979, ticks:500549
}
{
real_time:4261049.083437 global_time:578195, event:`SYSCALL: futex' (state:ENTERING_SYSCALL) tid:21979, ticks:500549
rax:0xffffffffffffffda rbx:0xca rcx:0xffffffffffffffff rdx:0x0 rsi:0x189 rdi:0x7efd4c4ba030 rbp:0x0 rsp:0x7efd4b6b1de0 r8:0x0 r9:0xffffffff r10:0x7efd4beb0340 r11:0x246 r12:0x0 r13:0x7efd4b6b1fa0 r14:0x7efd4beb0340 r15:0x7efd4c4ba078 rip:0x70000002 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0xca fs_base:0x7efd4beb2700 gs_base:0x0
}
{
real_time:4261049.083456 global_time:578196, event:`SYSCALLBUF_RESET' tid:21979, ticks:500549
}
{
real_time:4261049.083624 global_time:578197, event:`SYSCALL: futex' (state:EXITING_SYSCALL) tid:21975, ticks:44869827
rax:0xfffffffffffffdfc rbx:0xca rcx:0xffffffffffffffff rdx:0x0 rsi:0x189 rdi:0x7ffe8807f490 rbp:0x0 rsp:0x681ffde0 r8:0x0 r9:0xffffffff r10:0x7ffe8807ec20 r11:0x246 r12:0x0 r13:0x681fffa0 r14:0x7ffe8807ec20 r15:0x7ffe8807f498 rip:0x70000002 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0xca fs_base:0x7efd4c6b7240 gs_base:0x0
}
{
real_time:4261049.083831 global_time:578198, event:`SIGNAL: SIGABRT(async)' tid:21975, ticks:44869827
rax:0xfffffffffffffdfc rbx:0xca rcx:0xffffffffffffffff rdx:0x0 rsi:0x189 rdi:0x7ffe8807f490 rbp:0x0 rsp:0x681ffde0 r8:0x0 r9:0xffffffff r10:0x7ffe8807ec20 r11:0x246 r12:0x0 r13:0x681fffa0 r14:0x7ffe8807ec20 r15:0x7ffe8807f498 rip:0x70000002 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0xca fs_base:0x7efd4c6b7240 gs_base:0x0
}
{
real_time:4261049.083980 global_time:578199, event:`SIGNAL_DELIVERY: SIGABRT(async)' tid:21975, ticks:44869827
rax:0xfffffffffffffdfc rbx:0xca rcx:0xffffffffffffffff rdx:0x0 rsi:0x189 rdi:0x7ffe8807f490 rbp:0x0 rsp:0x681ffde0 r8:0x0 r9:0xffffffff r10:0x7ffe8807ec20 r11:0x246 r12:0x0 r13:0x681fffa0 r14:0x7ffe8807ec20 r15:0x7ffe8807f498 rip:0x70000002 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0xca fs_base:0x7efd4c6b7240 gs_base:0x0
}
.... some similars of the following:
{
real_time:4261049.086925 global_time:578221, event:`EXIT' tid:22617, ticks:41302
}
{
real_time:4261049.087031 global_time:578222, event:`EXIT' tid:22612, ticks:2289779
}
{
real_time:4261049.087141 global_time:578223, event:`EXIT' tid:22564, ticks:97
}
{
real_time:4261049.087251 global_time:578224, event:`EXIT' tid:22561, ticks:711
}
{
real_time:4261049.087354 global_time:578225, event:`EXIT' tid:21985, ticks:3288283
}
{
real_time:4261049.087536 global_time:578226, event:`EXIT' tid:21979, ticks:500549
}
{
real_time:4261051.848308 global_time:578227, event:`SCHED' tid:21975, ticks:44869827
rax:0xfffffffffffffdfc rbx:0xca rcx:0xffffffffffffffff rdx:0x0 rsi:0x189 rdi:0x7ffe8807f490 rbp:0x0 rsp:0x681ffde0 r8:0x0 r9:0xffffffff r10:0x7ffe8807ec20 r11:0x246 r12:0x0 r13:0x681fffa0 r14:0x7ffe8807ec20 r15:0x7ffe8807f498 rip:0x70000002 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0xca fs_base:0x7efd4c6b7240 gs_base:0x0
}
{
real_time:4261051.848351 global_time:578228, event:`EXIT' tid:21975, ticks:44869827
could this be an assert in the arangodb process? Alternatively our testing framework sends a SIG_ABRT if the shutdown doesn't work. So, is it sent from outside in? or is it emitted internally?
rocallahan
commented
5 years ago It was emitted by rr code.
rocallahan
commented
5 years ago This is bad:
{
real_time:4261049.080672 global_time:578183, event:`SCHED' tid:21979, ticks:492881
rax:0x1c rbx:0xd0 rcx:0xffffffffffffffff rdx:0x4 rsi:0x3000 rdi:0x7efd4361e000 rbp:0x7efd4b2b2000 rsp:0x7efd4b6b1de0 r8:0x0 r9:0x0 r10:0x0 r11:0x246 r12:0x7efd4c4b3980 r13:0x7efd4b6b1fa0 r14:0x7efd4b2b20fe r15:0x4000 rip:0x70000006 eflags:0x246 cs:0x33 ss:0x2b ds:0x0 es:0x0 fs:0x0 gs:0x0 orig_rax:0xffffffffffffff09 fs_base:0x7efd4beb2700 gs_base:0x0
}We took a SCHED interrupt at IP 0x70000006. That's just before entering an untraced syscall. That shouldn't happen, we should never take a SCHED inside the buffered syscall critical sections.
rocallahan
commented
5 years ago RecordTask::is_in_syscallbuf looks right. RSP 0x7efd4b6b1de0 looks reasonable. It looks like we're about to call madvise(MADV_DONTNEED). The parameters look reasonable.
We should be hitting the code in is_in_syscallbuf that does
bool ok = true;
uint64_t addr = read_ptr(this, regs().sp(), &ok);
if (ok) {
p = addr;
}@dothebart did you build rr yourself? Can you extract the librrpreload.so that you built and put it somewhere we can look at it?
dothebart
commented
5 years ago yes, its been compiled from the state the day before I opened this report. I required a bugfix for a bug reported by @goedderz last June. find it over at https://docs.arangodb.com/librrpreload.so
khuey
commented
5 years ago I don't see anything obviously wrong with that binary.
Is it possible for you to rr pack the trace and upload it somewhere so we can examine it?
dothebart
commented
5 years ago Hi, please find it at: https://docs.arangodb.com/github-issue-2292.tar.bz2
khuey
commented
5 years ago Thank you. Unfortunately I can't replay it on my system for some reason. :(
Can you run rr replay -g 578182 <your trace> and then provide the stack of the currently running thread when gdb opens?
dothebart
commented
5 years ago Hi, its:
(rr) bt
#0 0x00007efd4d164d3d in _breakpoint_table_entry_end () from /usr/local/bin/../lib/rr/librrpreload.so
#1 0x00007efd4d1643fb in commit_raw_syscall (syscallno=<optimized out>, record_end=<optimized out>, ret=-22) at ../src/preload/syscallbuf.c:950
#2 0x00007efd4d15f931 in syscall_hook (call=0x7efd4b6b1fa0) at ../src/preload/syscallbuf.c:2896
#3 0x00007efd4d15f09a in _syscall_hook_trampoline () from /usr/local/bin/../lib/rr/librrpreload.so
#4 0x00007efd4d15f0ca in __morestack () from /usr/local/bin/../lib/rr/librrpreload.so
#5 0x00007efd4d15f0d0 in _syscall_hook_trampoline_48_3d_01_f0_ff_ff () from /usr/local/bin/../lib/rr/librrpreload.so
#6 0x00007efd4c931dfd in madvise () at ../sysdeps/unix/syscall-template.S:78
#7 0x000055aad9e191ae in je_pages_purge_lazy (addr=<optimized out>, size=<optimized out>) at src/pages.c:260
#8 0x000055aad9e13e3b in extent_purge_lazy_impl (growing_retained=false, length=<optimized out>, offset=0, extent=0x7efd4c0f0840, r_extent_hooks=0x7efd4beb0178, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at include/jemalloc/internal/extent_inlines.h:125
#9 je_extent_purge_lazy_wrapper (tsdn=tsdn@entry=0x7efd4beb0bf8, arena=arena@entry=0x7efd4c4b3980, r_extent_hooks=r_extent_hooks@entry=0x7efd4beb0178, extent=extent@entry=0x7efd4c0f0840, offset=offset@entry=0, length=<optimized out>) at src/extent.c:1725
#10 0x000055aad9ddf2c7 in arena_decay_stashed (is_background_thread=<optimized out>, decay_extents=<synthetic pointer>, all=<optimized out>, extents=0x7efd4c4b55e8, decay=0x7efd4c4b6a88, r_extent_hooks=0x7efd4beb0178, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at include/jemalloc/internal/extent_inlines.h:125
#11 arena_decay_to_limit (tsdn=tsdn@entry=0x7efd4beb0bf8, arena=arena@entry=0x7efd4c4b3980, decay=decay@entry=0x7efd4c4b6a88, extents=extents@entry=0x7efd4c4b55e8, all=all@entry=false, npages_limit=<optimized out>, is_background_thread=true) at src/arena.c:1010
#12 0x000055aad9ddf939 in arena_decay_try_purge (is_background_thread=<optimized out>, npages_limit=<optimized out>, current_npages=<optimized out>, extents=<optimized out>, decay=<optimized out>, arena=<optimized out>, tsdn=<optimized out>) at src/arena.c:694
#13 arena_maybe_decay (tsdn=tsdn@entry=0x7efd4beb0bf8, arena=arena@entry=0x7efd4c4b3980, decay=decay@entry=0x7efd4c4b6a88, extents=extents@entry=0x7efd4c4b55e8, is_background_thread=is_background_thread@entry=true) at src/arena.c:839
#14 0x000055aad9de189a in arena_maybe_decay (is_background_thread=true, extents=0x7efd4c4b6a88, decay=0x7efd4c4b6a88, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:798
#15 arena_decay_impl (all=<optimized out>, is_background_thread=true, extents=0x7efd4c4b6a88, decay=0x7efd4c4b6a88, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:1037
#16 arena_decay_dirty (all=<optimized out>, is_background_thread=true, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:1057
#17 je_arena_decay (tsdn=0x7efd4beb0bf8, arena=0x7efd4c4b3980, is_background_thread=true, all=<optimized out>) at src/arena.c:1070
#18 0x000055aad9de76b5 in background_work_sleep_once (ind=0, info=<optimized out>, tsdn=<optimized out>) at src/background_thread.c:294
#19 background_thread0_work (tsd=<optimized out>) at src/background_thread.c:450
#20 background_work (ind=<optimized out>, tsd=<optimized out>) at src/background_thread.c:483
#21 background_thread_entry () at src/background_thread.c:513
#22 0x00007efd4cd36f2a in start_thread (arg=0x7efd4beb2700) at pthread_create.c:463
#23 0x00007efd4c936edf in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
(rr)I guess its jemalloc modifying some memory mapping?
dothebart
commented
5 years ago Ok, my bad, I just learned that arangodb + jemalloc doesn't work with rr (and probably never will?)
khuey
commented
5 years ago I don't know of any reason that wouldn't work with rr, and we'd like to fix this.
At that moment above, what does gdb say for p $rsp and x/1xg $rsp?
dothebart
commented
5 years ago (rr) p $rsp
$1 = (void *) 0x7efd4beb04c0
(rr) x/1xg $rsp
0x7efd4beb04c0: 0x0000000000000000The program in question is
https://github.com/arangodb/arangodb/
compiled via
./scripts/build-deb.sh (requires cmake, python 2.7, recent GCC)
and the case is invoked by starting it as a cluster using
ln -s build-deb build
./scripts/unittest http_server --cluster true --rr true --test api-users-spec.rb
this will require ruby rspec to run, however in some cases it crashes when setting up the servers; this command should also get you there but not require rspec:
./scripts/unittest shell_server_aql --cluster true --rr true
khuey
commented
5 years ago Thanks for the instructions. I was able to successfully build arangodb and run the tests but I wasn't able to reproduce any problems replaying the (six) recordings of the arangodb nodes. This bug so far looks like a particularly nasty race, so that's not too surprising.
Unfortunately I gave you the wrong instructions before. Please run rr replay -g 578183 <your trace> (note the different event number), then make sure gdb is on thread 21979, and then give me the backtrace. (RIP there should be 0x70000006 or 0x70000008.) Also p $rsp and x/1xg $rsp again please.
Thanks so much for struggling through this with us :)
dothebart
commented
5 years ago Thanks a lot for RR! so struggling is more than naturally to be taken! (sorry somehow the github notification slippet through...) Its says there is no thread 21979 - however:
(rr) p $rsp
$1 = (void *) 0x7efd2e0f5e98
(rr)
$2 = (void *) 0x7efd2e0f5e98
(rr) x/1xg $rsp
0x7efd2e0f5e98: 0x00007efd4d1643fb
(rr) ```You can't just thread 21979 in gdb. You need to do info threads, which will give you output that looks something like
(rr) info threads
Id Target Id Frame
* 1 Thread 17584.17584 (mmap_hardlink_3) main () at /home/khuey/dev/rr/src/test/threads.c:57
3 Thread 17584.17585 (mmap_hardlink_3) 0x0000000070000002 in ?? ()
4 Thread 17584.17586 (mmap_hardlink_3) 0x0000000070000002 in ?? ()The format there is thread <gdb thread number>.
If there really is no matching tid, what does when output?
dothebart
commented
5 years ago Hm, there is that jemalloc-thread matching this:
(rr) info threads
Id Target Id Frame
1 Thread 21975.11866 (Scheduler) 0x0000000070000002 in ?? ()
2 Thread 21975.13766 (Scheduler) 0x0000000070000002 in ?? ()
3 Thread 21975.21975 (mmap_pack_17_arangod) 0x0000000070000002 in ?? ()
4 Thread 21975.21979 (jemalloc_bg_thd) 0x0000000070000006 in ?? () <--------
5 Thread 21975.21985 (Logging) 0x0000000070000002 in ?? ()
6 Thread 21975.22561 (ArangoSearch #0) 0x0000000070000002 in ?? ()
7 Thread 21975.22564 (ArangoSearch #1) 0x0000000070000002 in ?? ()
8 Thread 21975.22612 (SchedulerManage) 0x0000000070000002 in ?? ()
9 Thread 21975.22617 (CacheRebalancer) 0x0000000070000002 in ?? ()
10 Thread 21975.22638 (mmap_pack_17_arangod) 0x0000000070000002 in ?? ()
11 Thread 21975.22639 (mmap_pack_17_arangod) 0x0000000070000002 in ?? ()
12 Thread 21975.22642 (mmap_pack_17_arangod) 0x0000000070000002 in ?? ()
13 Thread 21975.22738 (MMFilesCleanup) 0x0000000070000002 in ?? ()
14 Thread 21975.22740 (DatabaseManager) 0x0000000070000002 in ?? ()
15 Thread 21975.22742 (WalAllocator) 0x0000000070000002 in ?? ()
16 Thread 21975.22744 (WalSynchronizer) 0x0000000070000002 in ?? ()
17 Thread 21975.22747 (WalCollector) 0x0000000070000002 in ?? ()
18 Thread 21975.22748 (WalRemover) 0x0000000070000002 in ?? ()
19 Thread 21975.22749 (FlushThread) 0x0000000070000002 in ?? ()
20 Thread 21975.22750 (MMFilesCompacto) 0x0000000070000002 in ?? ()
21 Thread 21975.22752 (ClusterComm) 0x0000000070000002 in ?? ()
22 Thread 21975.22755 (Heartbeat) 0x0000000070000002 in ?? ()
23 Thread 21975.22760 (MaintenanceWork) 0x0000000070000002 in ?? ()
24 Thread 21975.22762 (MaintenanceWork) 0x0000000070000002 in ?? ()
25 Thread 21975.22770 (Statistics) 0x0000000070000002 in ?? ()
26 Thread 21975.22771 (StatisticsWorke) 0x0000000070000002 in ?? ()
* 27 Thread 21975.22772 (Maintenance) 0x00007efd4d164d27 in _breakpoint_table_entry_end () from /usr/local/bin/../lib/rr/librrpreload.so
28 Thread 21975.27349 (Scheduler) 0x0000000070000002 in ?? ()looking into it:
(rr) thread 4
[Switching to thread 4 (Thread 21975.21979)]
#0 0x0000000070000006 in ?? ()
(rr) bt
#0 0x0000000070000006 in ?? ()
#1 0x00007efd4d164d15 in _raw_syscall () from /usr/local/bin/../lib/rr/librrpreload.so
#2 0x00007efd4d162333 in untraced_syscall_base (syscallno=28, a0=<optimized out>, a1=<optimized out>, a2=-1, a3=0, a4=0, a5=0, syscall_instruction=<optimized out>) at ../src/preload/syscallbuf.c:311
#3 sys_madvise (call=<optimized out>) at ../src/preload/syscallbuf.c:1648
#4 syscall_hook_internal (call=0x7efd4b6b1fa0) at ../src/preload/syscallbuf.c:2793
#5 0x00007efd4d15f931 in syscall_hook (call=0x7efd4b6b1fa0) at ../src/preload/syscallbuf.c:2896
#6 0x00007efd4d15f09a in _syscall_hook_trampoline () from /usr/local/bin/../lib/rr/librrpreload.so
#7 0x00007efd4d15f0ca in __morestack () from /usr/local/bin/../lib/rr/librrpreload.so
#8 0x00007efd4d15f0d0 in _syscall_hook_trampoline_48_3d_01_f0_ff_ff () from /usr/local/bin/../lib/rr/librrpreload.so
#9 0x00007efd4c931dfd in madvise () at ../sysdeps/unix/syscall-template.S:78
#10 0x000055aad9e191ce in je_pages_purge_forced (addr=<optimized out>, size=<optimized out>) at src/pages.c:280
#11 0x000055aad9e13b2e in je_extent_dalloc_wrapper (extent=0x7efd4c0f0840, r_extent_hooks=0x7efd4beb0178, arena=<optimized out>, tsdn=0x7efd4beb0bf8) at include/jemalloc/internal/extent_inlines.h:125
#12 je_extent_dalloc_wrapper (tsdn=tsdn@entry=0x7efd4beb0bf8, arena=arena@entry=0x7efd4c4b3980, r_extent_hooks=r_extent_hooks@entry=0x7efd4beb0178, extent=extent@entry=0x7efd4c0f0840) at src/extent.c:1521
#13 0x000055aad9ddf263 in arena_decay_stashed (is_background_thread=<optimized out>, decay_extents=<synthetic pointer>, all=<optimized out>, extents=0x7efd4c4b55e8, decay=0x7efd4c4b6a88, r_extent_hooks=0x7efd4beb0178, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:956
#14 arena_decay_to_limit (tsdn=tsdn@entry=0x7efd4beb0bf8, arena=arena@entry=0x7efd4c4b3980, decay=decay@entry=0x7efd4c4b6a88, extents=extents@entry=0x7efd4c4b55e8, all=all@entry=false, npages_limit=<optimized out>, is_background_thread=true) at src/arena.c:1010
#15 0x000055aad9ddf939 in arena_decay_try_purge (is_background_thread=<optimized out>, npages_limit=<optimized out>, current_npages=<optimized out>, extents=<optimized out>, decay=<optimized out>, arena=<optimized out>, tsdn=<optimized out>) at src/arena.c:694
#16 arena_maybe_decay (tsdn=tsdn@entry=0x7efd4beb0bf8, arena=arena@entry=0x7efd4c4b3980, decay=decay@entry=0x7efd4c4b6a88, extents=extents@entry=0x7efd4c4b55e8, is_background_thread=is_background_thread@entry=true) at src/arena.c:839
#17 0x000055aad9de189a in arena_maybe_decay (is_background_thread=true, extents=0x7efd4c4b6a88, decay=0x7efd4c4b6a88, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:798
#18 arena_decay_impl (all=<optimized out>, is_background_thread=true, extents=0x7efd4c4b6a88, decay=0x7efd4c4b6a88, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:1037
#19 arena_decay_dirty (all=<optimized out>, is_background_thread=true, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:1057
#20 je_arena_decay (tsdn=0x7efd4beb0bf8, arena=0x7efd4c4b3980, is_background_thread=true, all=<optimized out>) at src/arena.c:1070
#21 0x000055aad9de76b5 in background_work_sleep_once (ind=0, info=<optimized out>, tsdn=<optimized out>) at src/background_thread.c:294
#22 background_thread0_work (tsd=<optimized out>) at src/background_thread.c:450
#23 background_work (ind=<optimized out>, tsd=<optimized out>) at src/background_thread.c:483
#24 background_thread_entry () at src/background_thread.c:513
#25 0x00007efd4cd36f2a in start_thread (arg=0x7efd4beb2700) at pthread_create.c:463
#26 0x00007efd4c936edf in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
(rr) p $rsp
$3 = (void *) 0x7efd4b6b1de0
(rr) x/1xg $rsp
0x7efd4b6b1de0: 0x00007efd4d164d15thanks for explaining.
rocallahan
commented
5 years ago Thanks. That's all as expected, so I'm mystified.
khuey
commented
5 years ago At this point I think the possibilities are (in order of decreasing likelihood):
1) rr has the wrong location for the syscallbuf, so it incorrectly concludes this call is not in the syscallbuf. (i.e. we're deciding incorrectly at https://github.com/mozilla/rr/blob/4eac15b0c80c660a3a622cc0c1b1345f2246c0eb/src/RecordTask.cc#L1413) 2) A stray write has corrupted the syscallbuf state (in particular, corrupting the locked bit at https://github.com/mozilla/rr/blob/4eac15b0c80c660a3a622cc0c1b1345f2246c0eb/src/record_signal.cc#L571) could cause us to incorrectly deliver the signal. 3) A desched event is floating around somehow when it shouldn't be (https://github.com/mozilla/rr/blob/4eac15b0c80c660a3a622cc0c1b1345f2246c0eb/src/record_signal.cc#L550)
khuey
commented
5 years ago Ok, to rule out the first possibility:
rr replay -g 412 <your trace>
When gdb opens the thread should be in init_process in rr's syscallbuf.c. What are params.syscallbuf_code_start and params.syscallbuf_code_end? If 0x00007efd4d164d15 is not in that range that would explain things.
For the second:
rr replay -g 578183 <your trace>
Make sure you're on thread 21979 as we discussed above. Then p/x *thread_locals.
I don't know that any data would exist in replay that would allow us to rule out the third possibility.
dothebart
commented
5 years ago #0 0x0000000070000005 in ?? ()
No symbol table info available.
#1 0x00007efd4d164d15 in _raw_syscall () from /usr/local/bin/../lib/rr/librrpreload.so
No symbol table info available.
#2 0x00007efd4d15f287 in privileged_traced_syscall (a4=0, a5=0, syscallno=<optimized out>, a0=<optimized out>, a1=<optimized out>, a2=<optimized out>, a3=<optimized out>) at ../src/preload/syscallbuf.c:202
No locals.
#3 init_process () at ../src/preload/syscallbuf.c:663
syscall_patch_hooks = {{is_multi_instruction = 0 '\000', next_instruction_length = 6 '\006', next_instruction_bytes = "H=\001\360\377\377\000\000\000\000\000\000\000", hook_address = 139626385109195}, {is_multi_instruction = 0 '\000', next_instruction_length = 6 '\006', next_instruction_bytes = "H=\000\360\377\377\000\000\000\000\000\000\000", hook_address = 139626385109216}, {is_multi_instruction = 0 '\000', next_instruction_length = 4 '\004', next_instruction_bytes = "H\213<$\000\000\000\000\000\000\000\000\000", hook_address = 139626385109237}, {is_multi_instruction = 1 '\001', next_instruction_length = 3 '\003', next_instruction_bytes = "Z^\303\000\000\000\000\000\000\000\000\000\000", hook_address = 139626385109260}, {is_multi_instruction = 1 '\001', next_instruction_length = 4 '\004', next_instruction_bytes = "\211\302\367\332\000\000\000\000\000\000\000\000\000", hook_address = 139626385109278}, {is_multi_instruction = 1 '\001', next_instruction_length = 3 '\003', next_instruction_bytes = "\220\220\220\000\000\000\000\000\000\000\000\000\000", hook_address = 139626385109297}, {is_multi_instruction = 0 '\000', next_instruction_length = 5 '\005', next_instruction_bytes = "\272\001", '\000' <repeats 11 times>, hook_address = 139626385109315}, {is_multi_instruction = 1 '\001', next_instruction_length = 4 '\004', next_instruction_bytes = "\211\301\061\322\000\000\000\000\000\000\000\000\000", hook_address = 139626385109338}, {is_multi_instruction = 1 '\001', next_instruction_length = 9 '\t', next_instruction_bytes = "\303\017\037\204\000\000\000\000\000\000\000\000\000", hook_address = 139626385109360}, {is_multi_instruction = 1 '\001', next_instruction_length = 6 '\006', next_instruction_bytes = "\303\017\037D\000\000\000\000\000\000\000\000\000", hook_address = 139626385109360}, {is_multi_instruction = 1 '\001', next_instruction_length = 4 '\004', next_instruction_bytes = "\303\017\037\000\000\000\000\000\000\000\000\000\000", hook_address = 139626385109360}, {is_multi_instruction = 1 '\001', next_instruction_length = 3 '\003', next_instruction_bytes = "\303f\220\000\000\000\000\000\000\000\000\000\000", hook_address = 139626385109360}}
params = <optimized out>
#4 0x00007efd4d1a00ca in call_init (l=<optimized out>, argc=argc@entry=31, argv=argv@entry=0x7ffe8807f8a8, env=env@entry=0x7ffe8807f9a8) at dl-init.c:72
j = <optimized out>
jm = <optimized out>
addrs = <optimized out>
init_array = <optimized out>
#5 0x00007efd4d1a01d6 in call_init (env=0x7ffe8807f9a8, argv=0x7ffe8807f8a8, argc=31, l=<optimized out>) at dl-init.c:118
init_array = <optimized out>
init_array = <optimized out>
j = <optimized out>
jm = <optimized out>
addrs = <optimized out>
#6 _dl_init (main_map=0x7efd4d1ba170, argc=31, argv=0x7ffe8807f8a8, env=0x7ffe8807f9a8) at dl-init.c:119
preinit_array = <optimized out>
preinit_array_size = <optimized out>
i = 1
#7 0x00007efd4d19224a in _dl_start_user () from /lib64/ld-linux-x86-64.so.2
library_path = 0x0
version_info = 0
any_debug = 0
_dl_rtld_libname = {name = 0x55aad7d2a2e0 "/lib64/ld-linux-x86-64.so.2", next = 0x7efd4d1b9fe0 <newname>, dont_free = 0}
relocate_time = 51126894
_dl_rtld_libname2 = {name = 0x0, next = 0x0, dont_free = 0}
start_time = 17041870525661488
tls_init_tp_called = true
load_time = 620315066
audit_list = 0x0
preloadlist = 0x7ffe88080f17 "/usr/local/bin/../lib/rr/librrpreload.so:::"
__GI__dl_argv = 0x7ffe8807f8a8
_dl_argc = 31
audit_list_string = 0x0
_rtld_global = {_dl_ns = {{_ns_loaded = 0x7efd4d1ba170, _ns_nloaded = 17, _ns_main_searchlist = 0x7efd4d1ba428, _ns_global_scope_alloc = 0, _ns_unique_sym_table = {lock = {mutex = pthread_mutex_t = {Type = Recursive, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, entries = 0x7efd4c6b3000, size = 127, n_elements = 82, free = 0x7efd4d1a9eb0 <free>}, _ns_debug = {r_version = 0, r_map = 0x0, r_brk = 0, r_state = RT_CONSISTENT, r_ldbase = 0}}, {_ns_loaded = 0x0, _ns_nloaded = 0, _ns_main_searchlist = 0x0, _ns_global_scope_alloc = 0, _ns_unique_sym_table = {lock = {mutex = pthread_mutex_t = {Type = Normal, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, entries = 0x0, size = 0, n_elements = 0, free = 0x0}, _ns_debug = {r_version = 0, r_map = 0x0, r_brk = 0, r_state = RT_CONSISTENT, r_ldbase = 0}} <repeats 15 times>}, _dl_nns = 1, _dl_load_lock = {mutex = pthread_mutex_t = {Type = Recursive, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, _dl_load_write_lock = {mutex = pthread_mutex_t = {Type = Recursive, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, _dl_load_adds = 17, _dl_initfirst = 0x0, _dl_cpuclock_offset = 17041870526279136, _dl_profile_map = 0x0, _dl_num_relocations = 6309, _dl_num_cache_relocations = 6687, _dl_all_dirs = 0x7efd4d1bac90, _dl_rtld_map = {l_addr = 139626385313792, l_name = 0x55aad7d2a2e0 "/lib64/ld-linux-x86-64.so.2", l_ld = 0x7efd4d1b8e68, l_next = 0x0, l_prev = 0x7efd4c9fc580, l_real = 0x7efd4d1b99f0 <_rtld_global+2448>, l_ns = 0, l_libname = 0x7efd4d1ba030 <_dl_rtld_libname>, l_info = {0x0, 0x0, 0x7efd4d1b8ee8, 0x7efd4d1b8ed8, 0x7efd4d1b8e78, 0x7efd4d1b8e98, 0x7efd4d1b8ea8, 0x7efd4d1b8f18, 0x7efd4d1b8f28, 0x7efd4d1b8f38, 0x7efd4d1b8eb8, 0x7efd4d1b8ec8, 0x0, 0x0, 0x7efd4d1b8e68, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7efd4d1b8ef8, 0x0, 0x0, 0x7efd4d1b8f08, 0x0 <repeats 12 times>, 0x7efd4d1b8f58, 0x7efd4d1b8f48, 0x0, 0x0, 0x7efd4d1b8f78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7efd4d1b8f68, 0x0 <repeats 25 times>, 0x7efd4d1b8e88}, l_phdr = 0x7efd4d191040, l_entry = 0, l_phnum = 9, l_ldnum = 0, l_searchlist = {r_list = 0x0, r_nlist = 0}, l_symbolic_searchlist = {r_list = 0x0, r_nlist = 0}, l_loader = 0x0, l_versions = 0x7efd4c6b9bb0, l_nversions = 6, l_nbuckets = 17, l_gnu_bitmask_idxbits = 3, l_gnu_shift = 8, l_gnu_bitmask = 0x7efd4d191348, {l_gnu_buckets = 0x7efd4d191368, l_chain = 0x7efd4d191368}, {l_gnu_chain_zero = 0x7efd4d1913a8, l_buckets = 0x7efd4d1913a8}, l_direct_opencount = 0, l_type = lt_library, l_relocated = 1, l_init_called = 1, l_global = 1, l_reserved = 0, l_phdr_allocated = 0, l_soname_added = 0, l_faked = 0, l_need_tls_init = 0, l_auditing = 0, l_audit_any_plt = 0, l_removed = 0, l_contiguous = 0, l_symbolic_in_local_scope = 0, l_free_initfini = 0, l_rpath_dirs = {dirs = 0x0, malloced = 0}, l_reloc_result = 0x0, l_versyms = 0x7efd4d191984, l_origin = 0x0, l_map_start = 139626385313792, l_map_end = 139626385482096, l_text_end = 139626385439424, l_scope_mem = {0x0, 0x0, 0x0, 0x0}, l_scope_max = 0, l_scope = 0x0, l_local_scope = {0x0, 0x0}, l_file_id = {dev = 0, ino = 0}, l_runpath_dirs = {dirs = 0x0, malloced = 0}, l_initfini = 0x0, l_reldeps = 0x0, l_reldepsmax = 0, l_used = 1, l_feature_1 = 0, l_flags_1 = 0, l_flags = 0, l_idx = 0, l_mach = {plt = 0, gotplt = 0, tlsdesc_table = 0x0}, l_lookup_cache = {sym = 0x7efd4d1914f0, type_class = 1, value = 0x7efd4c9fc000, ret = 0x7efd4c844158}, l_tls_initimage = 0x0, l_tls_initimage_size = 0, l_tls_blocksize = 0, l_tls_align = 0, l_tls_firstbyte_offset = 0, l_tls_offset = 0, l_tls_modid = 0, l_tls_dtor_count = 0, l_relro_addr = 161408, l_relro_size = 2432, l_serial = 0, l_audit = 0x7efd4d1b9e60 <_rtld_global+3584>}, audit_data = {{cookie = 0, bindflags = 0} <repeats 16 times>}, _dl_rtld_lock_recursive = 0x7efd4cd395e0 <__GI___pthread_mutex_lock>, _dl_rtld_unlock_recursive = 0x7efd4cd3ad70 <__GI___pthread_mutex_unlock>, _dl_make_stack_executable_hook = 0x7efd4cd36300 <__make_stacks_executable>, _dl_stack_flags = 7, _dl_tls_dtv_gaps = false, _dl_tls_max_dtv_idx = 3, _dl_tls_dtv_slotinfo_list = 0x7efd4c6b9c40, _dl_tls_static_nelem = 3, _dl_tls_static_size = 11072, _dl_tls_static_used = 7104, _dl_tls_static_align = 64, _dl_initial_dtv = 0x7efd4c6b7ba0, _dl_tls_generation = 1, _dl_init_static_tls = 0x7efd4cd36a10 <__pthread_init_static_tls>, _dl_wait_lookup_done = 0x7efd4cd36b30 <__wait_lookup_done>, _dl_scope_free_list = 0x0}
_rtld_global_ro = {_dl_debug_mask = 0, _dl_osversion = 266752, _dl_platform = 0x7ffe8807fcb9 "x86_64", _dl_platformlen = 6, _dl_pagesize = 4096, _dl_inhibit_cache = 0, _dl_initial_searchlist = {r_list = 0x7efd4c9fcdf8, r_nlist = 16}, _dl_clktck = 100, _dl_verbose = 0, _dl_debug_fd = 2, _dl_lazy = 1, _dl_bind_not = 0, _dl_dynamic_weak = 0, _dl_fpu_control = 895, _dl_correct_cache_id = 771, _dl_hwcap = 2, _dl_auxv = 0x7ffe8807fb68, _dl_x86_cpu_features = {kind = arch_kind_intel, max_cpuid = 13, cpuid = {{eax = 132823, ebx = 119539712, ecx = 532603839, edx = 3219913727}, {eax = 0, ebx = 0, ecx = 0, edx = 0}, {eax = 0, ebx = 0, ecx = 1, edx = 739248128}}, family = 6, model = 45, xsave_state_size = 896, xsave_state_full_size = 896, feature = {1359985}, data_cache_size = 0, shared_cache_size = 0, non_temporal_threshold = 0}, _dl_x86_hwcap_flags = {"sse2\000\000\000\000", "x86_64\000\000", "avx512_1"}, _dl_x86_platforms = {"i586\000\000\000\000", "i686\000\000\000\000", "haswell\000", "xeon_phi"}, _dl_inhibit_rpath = 0x0, _dl_origin_path = 0x0, _dl_use_load_bias = 0, _dl_profile = 0x0, _dl_profile_output = 0x7efd4d1b007c "/var/tmp", _dl_trace_prelink = 0x0, _dl_trace_prelink_map = 0x0, _dl_init_all_dirs = 0x7efd4d1bac90, _dl_sysinfo_dso = 0x7ffe88157000, _dl_sysinfo_map = 0x7efd4d1ba710, _dl_hwcap2 = 0, _dl_debug_printf = 0x7efd4d1a0db0 <_dl_debug_printf>, _dl_mcount = 0x7efd4d1a2140 <__GI__dl_mcount>, _dl_lookup_symbol_x = 0x7efd4d19b0d0 <_dl_lookup_symbol_x>, _dl_check_caller = 0x7efd4d1a3530 <_dl_check_caller>, _dl_open = 0x7efd4d1a3a60 <_dl_open>, _dl_close = 0x7efd4d1a5810 <_dl_close>, _dl_tls_get_addr_soft = 0x7efd4d1a3090 <_dl_tls_get_addr_soft>, _dl_discover_osversion = 0x7efd4d1a9aa0 <_dl_discover_osversion>, _dl_audit = 0x0, _dl_naudit = 0}
_dl_skip_args = 0
__pointer_chk_guard_local = 6182949168329942519
#8 0x000000000000001f in ?? ()
(rr) frame 3
#3 init_process () at ../src/preload/syscallbuf.c:663
663 privileged_traced_syscall1(SYS_rrcall_init_preload, ¶ms);
(rr) print params.syscallbuf_code_start
$1 = <optimized out>
(rr) print params.syscallbuf_code_end
$2 = <optimized out>
:-(regarding the second:
(rr) thread 4
[Switching to thread 4 (Thread 21975.21979)]
(rr) bt full
#0 0x0000000070000006 in ?? ()
No symbol table info available.
#1 0x00007efd4d164d15 in _raw_syscall () from /usr/local/bin/../lib/rr/librrpreload.so
No symbol table info available.
#2 0x00007efd4d162333 in untraced_syscall_base (syscallno=28, a0=<optimized out>, a1=<optimized out>, a2=-1, a3=0, a4=0, a5=0, syscall_instruction=<optimized out>) at ../src/preload/syscallbuf.c:311
rec = <optimized out>
ret = <optimized out>
tmp_in_replay = <optimized out>
#3 sys_madvise (call=<optimized out>) at ../src/preload/syscallbuf.c:1648
length = <optimized out>
advice = <optimized out>
ptr = 0x7efd4b2b20fe
ret = <optimized out>
addr = <optimized out>
s = <optimized out>
#4 syscall_hook_internal (call=0x7efd4b6b1fa0) at ../src/preload/syscallbuf.c:2793
No locals.
#5 0x00007efd4d15f931 in syscall_hook (call=0x7efd4b6b1fa0) at ../src/preload/syscallbuf.c:2896
result = <optimized out>
#6 0x00007efd4d15f09a in _syscall_hook_trampoline () from /usr/local/bin/../lib/rr/librrpreload.so
No symbol table info available.
#7 0x00007efd4d15f0ca in __morestack () from /usr/local/bin/../lib/rr/librrpreload.so
No symbol table info available.
#8 0x00007efd4d15f0d0 in _syscall_hook_trampoline_48_3d_01_f0_ff_ff () from /usr/local/bin/../lib/rr/librrpreload.so
No symbol table info available.
#9 0x00007efd4c931dfd in madvise () at ../sysdeps/unix/syscall-template.S:78
No locals.
#10 0x000055aad9e191ce in je_pages_purge_forced (addr=<optimized out>, size=<optimized out>) at src/pages.c:280
No locals.
#11 0x000055aad9e13b2e in je_extent_dalloc_wrapper (extent=0x7efd4c0f0840, r_extent_hooks=0x7efd4beb0178, arena=<optimized out>, tsdn=0x7efd4beb0bf8) at include/jemalloc/internal/extent_inlines.h:125
zeroed = <optimized out>
zeroed = <optimized out>
#12 je_extent_dalloc_wrapper (tsdn=tsdn@entry=0x7efd4beb0bf8, arena=arena@entry=0x7efd4c4b3980, r_extent_hooks=r_extent_hooks@entry=0x7efd4beb0178, extent=extent@entry=0x7efd4c0f0840) at src/extent.c:1521
zeroed = <optimized out>
#13 0x000055aad9ddf263 in arena_decay_stashed (is_background_thread=<optimized out>, decay_extents=<synthetic pointer>, all=<optimized out>, extents=0x7efd4c4b55e8, decay=0x7efd4c4b6a88, r_extent_hooks=0x7efd4beb0178, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:956
npages = 3
extent = 0x7efd4c0f0840
nmadvise = 5
nunmapped = 12
npurged = 15
muzzy_decay_ms = <optimized out>
nmadvise = <optimized out>
nunmapped = <optimized out>
npurged = <optimized out>
muzzy_decay_ms = <optimized out>
extent = <optimized out>
npages = <optimized out>
#14 arena_decay_to_limit (tsdn=tsdn@entry=0x7efd4beb0bf8, arena=arena@entry=0x7efd4c4b3980, decay=decay@entry=0x7efd4c4b6a88, extents=extents@entry=0x7efd4c4b55e8, all=all@entry=false, npages_limit=<optimized out>, is_background_thread=true) at src/arena.c:1010
npurged = <optimized out>
extent_hooks = 0x55aadab55840 <je_extent_hooks_default>
decay_extents = {qlh_first = 0x7efd4c0f5ac0}
npurge = <optimized out>
#15 0x000055aad9ddf939 in arena_decay_try_purge (is_background_thread=<optimized out>, npages_limit=<optimized out>, current_npages=<optimized out>, extents=<optimized out>, decay=<optimized out>, arena=<optimized out>, tsdn=<optimized out>) at src/arena.c:694
No locals.
#16 arena_maybe_decay (tsdn=tsdn@entry=0x7efd4beb0bf8, arena=arena@entry=0x7efd4c4b3980, decay=decay@entry=0x7efd4c4b6a88, extents=extents@entry=0x7efd4c4b55e8, is_background_thread=is_background_thread@entry=true) at src/arena.c:839
decay_ms = <optimized out>
time = {ns = 4261049077121149}
advance_epoch = false
#17 0x000055aad9de189a in arena_maybe_decay (is_background_thread=true, extents=0x7efd4c4b6a88, decay=0x7efd4c4b6a88, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:798
decay_ms = <optimized out>
time = <optimized out>
advance_epoch = <optimized out>
decay_ms = <optimized out>
time = <optimized out>
advance_epoch = <optimized out>
#18 arena_decay_impl (all=<optimized out>, is_background_thread=true, extents=0x7efd4c4b6a88, decay=0x7efd4c4b6a88, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:1037
epoch_advanced = <optimized out>
npages_new = <optimized out>
epoch_advanced = <optimized out>
npages_new = <optimized out>
#19 arena_decay_dirty (all=<optimized out>, is_background_thread=true, arena=0x7efd4c4b3980, tsdn=0x7efd4beb0bf8) at src/arena.c:1057
No locals.
#20 je_arena_decay (tsdn=0x7efd4beb0bf8, arena=0x7efd4c4b3980, is_background_thread=true, all=<optimized out>) at src/arena.c:1070
No locals.
#21 0x000055aad9de76b5 in background_work_sleep_once (ind=0, info=<optimized out>, tsdn=<optimized out>) at src/background_thread.c:294
arena = 0x7efd4c4b3980
interval = <optimized out>
i = 0
min_interval = 18446744073709551615
narenas = 1
min_interval = <optimized out>
narenas = <optimized out>
i = <optimized out>
arena = <optimized out>
interval = <optimized out>
#22 background_thread0_work (tsd=<optimized out>) at src/background_thread.c:450
created_threads = 0x7efd4beb02a0
i = 1
n_created = 1
created_threads = <optimized out>
i = <optimized out>
n_created = <optimized out>
info = <optimized out>
#23 background_work (ind=<optimized out>, tsd=<optimized out>) at src/background_thread.c:483
info = <optimized out>
info = <optimized out>
#24 background_thread_entry () at src/background_thread.c:513
thread_ind = <optimized out>
#25 0x00007efd4cd36f2a in start_thread (arg=0x7efd4beb2700) at pthread_create.c:463
pd = 0x7efd4beb2700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139626365527808, 9152745134204300188, 140731180643582, 140731180643583, 140731180643584, 0, -9007336819256284260, -9007326170654790756}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
#26 0x00007efd4c936edf in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
No locals.
(rr) frame 25
(rr) p *thread_locals
No symbol "thread_locals" in current context.
(rr) x *thread_locals
No symbol "thread_locals" in current context.so no luck over here either :-(
khuey
commented
5 years ago Ok, for the first one, lets look at things from rr's side instead. Run gdb --args rr replay <your trace> -a (-a tells rr not to open its own gdb) and break at https://github.com/mozilla/rr/blob/4eac15b0c80c660a3a622cc0c1b1345f2246c0eb/src/Task.cc#L2346 and inspect the struct from there.
dothebart
commented
5 years ago gdb> print *t
$2 = {_vptr$Task = 0x8970a0 <vtable for rr::ReplayTask+16>, unstable = false, stable_exit = false, scratch_ptr = {ptr = 1744830464}, scratch_size = 2097152, desched_fd_child = -1, cloned_file_data_fd_child = -1, hpc = {counting_period = 1152921504606846976, tid = 32719, fd_ticks_measure = {fd = 10}, fd_minus_ticks_measure = {fd = -1}, fd_ticks_interrupt = {fd = 8}, fd_ticks_in_transaction = {fd = -1}, fd_useless_counter = {fd = -1}, ticks_semantics_ = rr::TICKS_RETIRED_CONDITIONAL_BRANCHES, started = true, counting = false}, tid = 32719, rec_tid = 21975, syscallbuf_size = 0, syscallbuf_child = {ptr = 0}, stopping_breakpoint_table = {ptr = 0}, stopping_breakpoint_table_entry_size = 0, preload_globals = {ptr = 139626385277032}, thread_locals = "\000\000 h", '\000' <repeats 99 times>, serial = 1, as = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::AddressSpace*, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::AddressSpace*, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::AddressSpace> (use count 2, weak count 0) = {get() = 0x9e93e0}, fds = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::FdTable*, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::FdTable*, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::FdTable> (use count 2, weak count 0) = {get() = 0x9e7a70}, prname = <incomplete type>, ticks = 1542558, registers = {arch_ = rr::x86_64, u = {x86regs = {ebx = 0, ecx = 0, edx = 1293450512, esi = 32509, edi = -2012743256, ebp = 32766, eax = -2012743512, xds = 32766, xes = 31, xfs = 0, xgs = -2012743936, orig_eax = 32766, eip = 582, xcs = 0, eflags = 0, esp = 0, xss = 0}, x64regs = {r15 = 0, r14 = 139626385276176, r13 = 140731180644776, r12 = 140731180644520, rbp = 31, rbx = 140731180644096, r11 = 582, r10 = 0, r9 = 0, r8 = 0, rax = 18446744073709551578, rcx = 18446744073709551615, rdx = 0, rsi = 0, rdi = 140731180644008, orig_rax = 442, rip = 1879048197, cs = 51, eflags = 582, rsp = 140731180643936, ss = 43, fs_base = 139626373935680, gs_base = 0, ds = 0, es = 0, fs = 0, gs = 0}}}, address_of_last_execution_resume = {ptr = 139626380780374}, how_last_execution_resumed = rr::RESUME_SYSEMU, last_resume_orig_cx = 0, is_stopped = true, seccomp_bpf_enabled = false, detected_unexpected_exit = false, registers_dirty = true, extra_registers = {format_ = rr::ExtraRegisters::XSAVE, arch_ = rr::x86_64, data_ = std::vector of length 832, capacity 832 = {127 '\177', 3 '\003', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 128 '\200', 31 '\037', 0 '\000', 0 '\000', 255 '\377', 255 '\377', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000', 0 '\000'...}}, extra_registers_known = false, session_ = 0x9e6460, tg = warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::ThreadGroup*, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_ptr<rr::ThreadGroup*, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<rr::ThreadGroup> (use count 2, weak count 0) = {get() = 0x9e70c0}, thread_areas_ = std::vector of length 0, capacity 0, top_of_stack = {ptr = 0}, wait_status = {status = 1407}, pending_siginfo = {si_signo = 5, si_errno = 0, si_code = 128, __pad0 = 0, _sifields = {_pad = {0 <repeats 28 times>}, _kill = {si_pid = 0, si_uid = 0}, _timer = {si_tid = 0, si_overrun = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}}, _rt = {si_pid = 0, si_uid = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}}, _sigchld = {si_pid = 0, si_uid = 0, si_status = 0, si_utime = 0, si_stime = 0}, _sigfault = {si_addr = 0x0, si_addr_lsb = 0, _bounds = {_addr_bnd = {_lower = 0x0, _upper = 0x0}, _pkey = 0}}, _sigpoll = {si_band = 0, si_fd = 0}, _sigsys = {_call_addr = 0x0, _syscall = 0, _arch = 0}}}, seen_ptrace_exit_event = false, properties_ = {values = std::vector of length 0, capacity 0}, expecting_ptrace_interrupt_stop = 0}like that?
khuey
commented
5 years ago Sorry, I should have been more clear. "The struct" means params here.
dothebart
commented
5 years ago $3 = {syscallbuf_enabled = 1, syscall_patch_hook_count = 12, syscall_patch_hooks = {val = 140731180644096}, syscallhook_vsyscall_entry = {val = 0}, syscallbuf_code_start = {val = 139626385109120}, syscallbuf_code_end = {val = 139626385263907}, get_pc_thunks_start = {val = 0}, get_pc_thunks_end = {val = 0}, syscallbuf_final_exit_instruction = {val = 139626385109182}, globals = {val = 139626385277032}, breakpoint_table = {val = 139626385132835}, breakpoint_table_entry_size = 1}Great. According to that syscallbuf_code_start=0x7EFD4D15F080 and syscallbuf_code_end=7EFD4D184D23 and 0x00007efd4d164d15 is in range.
I'll have to think about how to poke at the second one from the other side.
I see a similar assertion triggered as in https://github.com/mozilla/rr/issues/2286 during working with ArangoDB - this is the transcript of the rr session when it goes bad:
The backtrace from gdb: