attermann
commented
7 years ago Here's another interesting crash. I took a look at the memory location that was supposed to be pointing to the "cbp" callback object in run_trans_callbacks and noticed that there was a string stored there instead. The string "sip:sip.outboundproxy.com" was assigned to the $ru variable in my script when the REGISTER request was received. Hopefully this provides a clue as to where memory overwriting/corruption may be occurring.
Backtrace:
[New LWP 10254]
warning: Can't read pathname for load map: Input/output error.
warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fffd1fba000
Core was generated by `/sbin/opensips -P /var/run/opensips/opensips.pid -m 256 -M 32 -u opensips -g op'.
Program terminated with signal 11, Segmentation fault.
#0 0x00007f36e2888361 in run_trans_callbacks (type=2, trans=0x7f36e339f318, req=<optimized out>, rpl=<optimized out>, code=<optimized out>) at t_hooks.c:209
209 cbp->callback( trans, type, ¶ms );
Thread 1 (LWP 10254):
#0 0x00007f36e2888361 in run_trans_callbacks (type=2, trans=0x7f36e339f318, req=<optimized out>, rpl=<optimized out>, code=<optimized out>) at t_hooks.c:209
params = {req = 0x7f36e32dc948, rpl = 0x7f36f2f23bd8, code = 200, param = 0x7f36e3349d78, extra1 = 0x0, extra2 = 0x0}
cbp = 0x7f36e3349d68
backup = 0x88b9e8
trans_backup = 0x7f36e339f318
__FUNCTION__ = "run_trans_callbacks"
#1 0x00007f36e288fc3c in t_reply_matching (p_msg=0x7f36f2f23bd8, p_branch=0x7fffd1f26f90) at t_lookup.c:843
p_cell = <optimized out>
hash_index = <optimized out>
entry_label = <optimized out>
branch_id = <optimized out>
hashi = <optimized out>
branchi = <optimized out>
p = <optimized out>
hashl = <optimized out>
branchl = <optimized out>
scan_space = <optimized out>
cseq = 0x7f36f2f25320
loopi = 0x0
loopl = <optimized out>
syni = <optimized out>
synl = <optimized out>
__FUNCTION__ = "t_reply_matching"
#2 0x00007f36e28902f5 in t_check (p_msg=0x7f36f2f23bd8, param_branch=<optimized out>) at t_lookup.c:918
local_branch = 232
__FUNCTION__ = "t_check"
#3 0x00007f36e28b12a3 in reply_received (p_msg=0x7f36f2f23bd8) at t_reply.c:1416
msg_status = <optimized out>
last_uac_status = <optimized out>
branch = <optimized out>
reply_status = <optimized out>
timer = <optimized out>
cancel_bitmap = <optimized out>
uac = <optimized out>
t = <optimized out>
backup_list = <optimized out>
has_reply_route = <optimized out>
__FUNCTION__ = "reply_received"
#4 0x000000000044c389 in forward_reply (msg=0x7f36f2f23bd8) at forward.c:495
new_buf = 0x0
to = 0x0
new_len = <optimized out>
mod = 0x7f36f2ef42c0
proto = <optimized out>
id = 0
send_sock = <optimized out>
s = <optimized out>
len = <optimized out>
__FUNCTION__ = "forward_reply"
#5 0x00000000004aade5 in receive_msg (buf=0x8ad080 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 70.42.44.204:6050;branch=z9hG4bK6d7d.9735cd23.0;i=838fc5c5;received=70.42.44.204\r\nVia: SIP/2.0/TCP 70.42.44.203:54300;received=70.42.44.203;branch=z9hG4bKtTNNvNgoNn9Mm"..., len=<optimized out>, rcv_info=<optimized out>, existing_context=0x0) at receive.c:257
ctx = 0x7f36f2f24808
msg = 0x7f36f2f23bd8
start = {tv_sec = 8827968, tv_usec = 139873717171744}
rc = 3
tmp = 0x0
in_buff = {s = 0x8ad080 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 70.42.44.204:6050;branch=z9hG4bK6d7d.9735cd23.0;i=838fc5c5;received=70.42.44.204\r\nVia: SIP/2.0/TCP 70.42.44.203:54300;received=70.42.44.203;branch=z9hG4bKtTNNvNgoNn9Mm"..., len = 612}
__FUNCTION__ = "receive_msg"
#6 0x00000000005f6c9b in udp_read_req (si=<optimized out>, bytes_read=<optimized out>) at net/proto_udp/proto_udp.c:192
ri = {src_ip = {af = 2, len = 4, u = {addrl = {139870680015435, 6425680}, addr32 = {775053899, 32566, 6425680, 0}, addr16 = {25163, 11826, 32566, 0, 3152, 98, 0, 0}, addr = "Kb2.6\177\000\000P\fb\000\000\000\000"}}, dst_ip = {af = 2, len = 4, u = {addrl = {3425446470, 0}, addr32 = {3425446470, 0, 0, 0}, addr16 = {10822, 52268, 0, 0, 0, 0, 0, 0}, addr = "F*,\314", '\000' <repeats 11 times>}}, src_port = 5060, dst_port = 6050, proto = 1, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {sa_family = 2, sa_data = "\023\304Kb2.\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195, sin_addr = {s_addr = 775053899}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 775053899, sin6_addr = {__in6_u = {__u6_addr8 = "\000\000\000\000\000\000\000\000\260k\361\362\066\177\000", __u6_addr16 = {0, 0, 0, 0, 27568, 62193, 32566, 0}, __u6_addr32 = {0, 0, 4075908016, 32566}}}, sin6_scope_id = 5201947}}, bind_address = 0x7f36f2ef36e0}
len = <optimized out>
buf = "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 70.42.44.204:6050;branch=z9hG4bK6d7d.9735cd23.0;i=838fc5c5;received=70.42.44.204\r\nVia: SIP/2.0/TCP 70.42.44.203:54300;received=70.42.44.203;branch=z9hG4bKtTNNvNgoNn9Mm"...
tmp = 0x0
fromlen = 16
p = <optimized out>
msg = {s = 0x8ad080 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 70.42.44.204:6050;branch=z9hG4bK6d7d.9735cd23.0;i=838fc5c5;received=70.42.44.204\r\nVia: SIP/2.0/TCP 70.42.44.203:54300;received=70.42.44.203;branch=z9hG4bKtTNNvNgoNn9Mm"..., len = 612}
__FUNCTION__ = "udp_read_req"
#7 0x00000000005d8eec in handle_io (fm=0x7f36f2f16c10, idx=<optimized out>, event_type=<optimized out>) at net/net_udp.c:259
read = 32512
#8 io_wait_loop_epoll (repeat=0, t=1, h=<optimized out>) at net/../io_wait_loop.h:284
ret = 1
n = 1
r = 0
i = <optimized out>
e = 0x7f36f2f16c10
ep_event = {events = 4119470451, data = {ptr = 0x4e164400007f36, fd = 32566, u32 = 32566, u64 = 21979529497050934}}
fd = <optimized out>
#9 0x00000000005dc85f in udp_start_processes (chd_rank=<optimized out>, startup_done=0x0) at net/net_udp.c:389
si = <optimized out>
load_p = 0x7f36e3208b20
pid = <optimized out>
i = <optimized out>
__FUNCTION__ = "udp_start_processes"
#10 0x000000000041d4c5 in main_loop () at main.c:677
startup_done = 0x0
chd_rank = 1
#11 main (argc=<optimized out>, argv=<optimized out>) at main.c:1283
cfg_stream = <optimized out>
c = <optimized out>
r = <optimized out>
tmp = 0x7fffd1f27e95 ""
tmp_len = <optimized out>
port = <optimized out>
proto = -772639260
protos_no = <optimized out>
options = 0x6116e8 "f:cCm:M:b:l:n:N:rRvdDFETSVhw:t:u:g:P:G:W:o:"
ret = -1
seed = 2401784065
rfd = 4
__FUNCTION__ = "main"Memory (around cbp at 0x7f36e3349d68)
(gdb) x /20cb 0x7f36e3349d54
0x7f36e3349d54: 0 '\000' 0 '\000' 0 '\000' 0 '\000' -48 '\320' -74 '\266' 61 '=' -29 '\343'
0x7f36e3349d5c: 54 '6' 127 '\177' 0 '\000' 0 '\000' 0 '\0000 '\000' 0 '\000' 0 '\000'
0x7f36e3349d64: 0 '\000' 0 '\000' 0 '\000' 0 '\000'
(gdb)
0x7f36e3349d68: 115 's' 105 'i' 112 'p' 58 ':' 115 's' 105 'i' 112 'p' 46 '.'
0x7f36e3349d70: 111 'o' 117 'u' 116 't' 98 'b' 111 'o' 117 'u' 110 'n' 100 'd'
0x7f36e3349d78: 112 'p' 114 'r' 111 'o' 120 'x'
(gdb)
0x7f36e3349d7c: 121 'y' 46 '.' 99 'c' 111 'o' 109 'm' 0 '\000' 0 '\0000 '\000'
0x7f36e3349d84: 0 '\000' 0 '\000' 0 '\000' 0 '\000' -96 '\240' -86 '\252' 65 'A' -29 '\343'
0x7f36e3349d8c: 54 '6' 127 '\177' 0 '\000' 0 '\000'
(gdb)
opensips 2.3 is frequently crashing in various places in code, but most appear related to the mid-registrar transaction callback mid_reg_resp_in. It looks like it could be memory corruption or something related which is beyond my experience with opensips. Details of my build and config below, as well as GDB and some debug log output for four separate crashes. Unfortunately the debug logs appear to be missing from immediately before some of the segfaults (caching?), but including anyway in case they are of any help.
CRASH # 1
CRASH # 2
CRASH # 3
CRASH # 4