vincentfretin
commented
1 week ago The leak is not consistent, it can also happen with
docker run --net=host -e MESSAGE_THREADS=1 janus:latest
and the old libnice 2020-07-06 13:53 (post 0.1.18) 48dac0d702b134f7b11b92602c234ba1120cc75b
but I reproduce it maybe more often with
docker run --net=host -e EVENT_LOOPS=4 -e MESSAGE_THREADS=1 janus:latest
and latest libnice post 0.1.22 55ff684d0989ffd65d02fafe1d2a43309576b8fa
I reproduce it with 2, 3 or 4 users in the room.
There is definitely no Arc anymore in switchboard when that happens.
[Mon Sep 16 14:02:47 2024] [WSS-0x518000011880] Destroying WebSocket client
[Mon Sep 16 14:02:47 2024] Destroying session 6956378221997667; 0x5070000592d0
[Mon Sep 16 14:02:47 2024] Detaching handle from Janus SFU plugin; 0x513000020640 0x504000010490 0x513000020640 0x50200006e5b0
[Mon Sep 16 14:02:47 2024] Detaching handle from Janus SFU plugin; 0x5130000209c0 0x504000010610 0x5130000209c0 0x50200008d550
[Mon Sep 16 14:02:47 2024] Detaching handle from Janus SFU plugin; 0x513000020b80 0x5040000106d0 0x513000020b80 0x50200008d650
[Mon Sep 16 14:02:47 2024] Hanging up WebRTC media on 0x5040000106d0.
[Mon Sep 16 14:02:47 2024] Hanging up WebRTC media on 0x504000010610.
[Mon Sep 16 14:02:47 2024] Hanging up WebRTC media on 0x504000010490.
[Mon Sep 16 14:02:47 2024] [2223840201439799] WebRTC resources freed; 0x513000020b80 0x5070000592d0
[Mon Sep 16 14:02:47 2024] [8826719679928899] WebRTC resources freed; 0x5130000209c0 0x5070000592d0
[Mon Sep 16 14:02:47 2024] [1540443264947778] WebRTC resources freed; 0x513000020640 0x5070000592d0
[Mon Sep 16 14:02:47 2024] Destroying SFU session 0x504000010490...
[Mon Sep 16 14:02:47 2024] Destroying SFU session 0x5040000106d0...
[Mon Sep 16 14:02:47 2024] Destroying SFU session 0x504000010610...
[Mon Sep 16 14:02:47 2024] Sessions:
Session 0x50e0000600d0
Session 0x50e000060990
Session 0x50e000060ed0
Session 0x50e000061410
Session 0x50e000062750
Session 0x50e0000630f0
Session 0x50e0000632b0
Session 0x50e000064430
Publishers by Room:
Room forest:
Publisher 0x50e0000600d0
Publisher 0x50e000060990
Publishers by User:
User 3103067687: Publisher 0x50e0000600d0
User 1246962283: Publisher 0x50e000060990
Subscribers by User:
User 336730444:
Subscriber 0x50e0000632b0
Subscriber 0x50e0000630f0
User 1246962283:
Subscriber 0x50e000061410
Subscriber 0x50e000064430
User 3103067687:
Subscriber 0x50e000060ed0
Subscriber 0x50e000062750
Publisher to Subscribers Mapping:
Forward Mapping:
Publisher 0x50e000060990:
Subscriber 0x50e000060ed0
Subscriber 0x50e0000630f0
Publisher 0x50e0000600d0:
Subscriber 0x50e000061410
Subscriber 0x50e0000632b0
Inverse Mapping:
Subscriber 0x50e000060ed0:
Publisher 0x50e000060990
Subscriber 0x50e000062750:
Subscriber 0x50e0000632b0:
Publisher 0x50e0000600d0
Subscriber 0x50e000064430:
Subscriber 0x50e000061410:
Publisher 0x50e0000600d0
Subscriber 0x50e0000630f0:
Publisher 0x50e000060990
Blockers to Miscreants Mapping:
Forward Mapping:
Inverse Mapping:
[Mon Sep 16 14:02:47 2024] [1540443264947778] Handle and related resources freed; 0x513000020640 0x5070000592d0
[Mon Sep 16 14:02:47 2024] Sessions:
Session 0x50e0000600d0
Session 0x50e000060990
Session 0x50e000060ed0
Session 0x50e000061410
Session 0x50e000062750
Session 0x50e0000630f0
Session 0x50e000064430
Publishers by Room:
Room forest:
Publisher 0x50e0000600d0
Publisher 0x50e000060990
Publishers by User:
User 3103067687: Publisher 0x50e0000600d0
User 1246962283: Publisher 0x50e000060990
Subscribers by User:
User 336730444:
Subscriber 0x50e0000630f0
User 1246962283:
Subscriber 0x50e000061410
Subscriber 0x50e000064430
User 3103067687:
Subscriber 0x50e000060ed0
Subscriber 0x50e000062750
Publisher to Subscribers Mapping:
Forward Mapping:
Publisher 0x50e000060990:
Subscriber 0x50e000060ed0
Subscriber 0x50e0000630f0
Publisher 0x50e0000600d0:
Subscriber 0x50e000061410
Inverse Mapping:
Subscriber 0x50e000060ed0:
Publisher 0x50e000060990
Subscriber 0x50e000062750:
Subscriber 0x50e000064430:
Subscriber 0x50e000061410:
Publisher 0x50e0000600d0
Subscriber 0x50e0000630f0:
Publisher 0x50e000060990
Blockers to Miscreants Mapping:
Forward Mapping:
Inverse Mapping:
[Mon Sep 16 14:02:47 2024] [2223840201439799] Handle and related resources freed; 0x513000020b80 0x5070000592d0
[Mon Sep 16 14:02:47 2024] Sessions:
Session 0x50e0000600d0
Session 0x50e000060990
Session 0x50e000060ed0
Session 0x50e000061410
Session 0x50e000062750
Session 0x50e000064430
Publishers by Room:
Room forest:
Publisher 0x50e0000600d0
Publisher 0x50e000060990
Publishers by User:
User 3103067687: Publisher 0x50e0000600d0
User 1246962283: Publisher 0x50e000060990
Subscribers by User:
User 1246962283:
Subscriber 0x50e000061410
Subscriber 0x50e000064430
User 3103067687:
Subscriber 0x50e000060ed0
Subscriber 0x50e000062750
Publisher to Subscribers Mapping:
Forward Mapping:
Publisher 0x50e000060990:
Subscriber 0x50e000060ed0
Publisher 0x50e0000600d0:
Subscriber 0x50e000061410
Inverse Mapping:
Subscriber 0x50e000060ed0:
Publisher 0x50e000060990
Subscriber 0x50e000062750:
Subscriber 0x50e000064430:
Subscriber 0x50e000061410:
Publisher 0x50e0000600d0
Blockers to Miscreants Mapping:
Forward Mapping:
Inverse Mapping:
[Mon Sep 16 14:02:47 2024] [8826719679928899] Handle and related resources freed; 0x5130000209c0 0x5070000592d0
[Mon Sep 16 14:02:47 2024] Hanging up WebRTC media on 0x504000010790.
[Mon Sep 16 14:02:47 2024] Hanging up WebRTC media on 0x504000010550.
[Mon Sep 16 14:02:47 2024] [1602784960739954] WebRTC resources freed; 0x513000020d40 0x507000016de0
[Mon Sep 16 14:02:47 2024] [5342214617638961] WebRTC resources freed; 0x513000020800 0x507000013810
[Mon Sep 16 14:02:48 2024] [WSS-0x518000010c80] Destroying WebSocket client
[Mon Sep 16 14:02:48 2024] Destroying session 367669542591413; 0x507000016de0
[Mon Sep 16 14:02:48 2024] Detaching handle from Janus SFU plugin; 0x513000020d40 0x504000010790 0x513000020d40 0x5020000a7730
[Mon Sep 16 14:02:48 2024] Detaching handle from Janus SFU plugin; 0x513000020480 0x504000010390 0x513000020480 0x50200004dbf0
[Mon Sep 16 14:02:48 2024] Detaching handle from Janus SFU plugin; 0x513000020100 0x504000010190 0x513000020100 0x502000038af0
[Mon Sep 16 14:02:48 2024] Destroying SFU session 0x504000010790...
[Mon Sep 16 14:02:48 2024] Hanging up WebRTC media on 0x504000010390.
[Mon Sep 16 14:02:48 2024] Hanging up WebRTC media on 0x504000010190.
[Mon Sep 16 14:02:48 2024] [2484182377087719] WebRTC resources freed; 0x513000020480 0x507000016de0
[Mon Sep 16 14:02:48 2024] [6967952292749315] WebRTC resources freed; 0x513000020100 0x507000016de0
[Mon Sep 16 14:02:48 2024] Sessions:
Session 0x50e0000600d0
Session 0x50e000060990
Session 0x50e000060ed0
Session 0x50e000061410
Session 0x50e000062750
Publishers by Room:
Room forest:
Publisher 0x50e0000600d0
Publisher 0x50e000060990
Publishers by User:
User 3103067687: Publisher 0x50e0000600d0
User 1246962283: Publisher 0x50e000060990
Subscribers by User:
User 1246962283:
Subscriber 0x50e000061410
User 3103067687:
Subscriber 0x50e000060ed0
Subscriber 0x50e000062750
Publisher to Subscribers Mapping:
Forward Mapping:
Publisher 0x50e000060990:
Subscriber 0x50e000060ed0
Publisher 0x50e0000600d0:
Subscriber 0x50e000061410
Inverse Mapping:
Subscriber 0x50e000060ed0:
Publisher 0x50e000060990
Subscriber 0x50e000062750:
Subscriber 0x50e000061410:
Publisher 0x50e0000600d0
Blockers to Miscreants Mapping:
Forward Mapping:
Inverse Mapping:
[Mon Sep 16 14:02:48 2024] [1602784960739954] Handle and related resources freed; 0x513000020d40 0x507000016de0
[Mon Sep 16 14:02:48 2024] Destroying SFU session 0x504000010390...
[Mon Sep 16 14:02:48 2024] Destroying SFU session 0x504000010190...
[Mon Sep 16 14:02:48 2024] Sessions:
Session 0x50e0000600d0
Session 0x50e000060990
Session 0x50e000060ed0
Session 0x50e000062750
Publishers by Room:
Room forest:
Publisher 0x50e0000600d0
Publisher 0x50e000060990
Publishers by User:
User 3103067687: Publisher 0x50e0000600d0
User 1246962283: Publisher 0x50e000060990
Subscribers by User:
User 3103067687:
Subscriber 0x50e000060ed0
Subscriber 0x50e000062750
Publisher to Subscribers Mapping:
Forward Mapping:
Publisher 0x50e000060990:
Subscriber 0x50e000060ed0
Publisher 0x50e0000600d0:
Inverse Mapping:
Subscriber 0x50e000060ed0:
Publisher 0x50e000060990
Subscriber 0x50e000062750:
Blockers to Miscreants Mapping:
Forward Mapping:
Inverse Mapping:
[Mon Sep 16 14:02:48 2024] [2484182377087719] Handle and related resources freed; 0x513000020480 0x507000016de0
[Mon Sep 16 14:02:48 2024] Sessions:
Session 0x50e0000600d0
Session 0x50e000060ed0
Session 0x50e000062750
Publishers by Room:
Room forest:
Publisher 0x50e0000600d0
Publishers by User:
User 3103067687: Publisher 0x50e0000600d0
Subscribers by User:
User 3103067687:
Subscriber 0x50e000060ed0
Subscriber 0x50e000062750
Publisher to Subscribers Mapping:
Forward Mapping:
Publisher 0x50e0000600d0:
Inverse Mapping:
Subscriber 0x50e000060ed0:
Subscriber 0x50e000062750:
Blockers to Miscreants Mapping:
Forward Mapping:
Inverse Mapping:
[Mon Sep 16 14:02:48 2024] [6967952292749315] Handle and related resources freed; 0x513000020100 0x507000016de0
[Mon Sep 16 14:02:48 2024] Hanging up WebRTC media on 0x504000010290.
[Mon Sep 16 14:02:48 2024] [1184320028341823] WebRTC resources freed; 0x5130000202c0 0x507000013810
[Mon Sep 16 14:02:48 2024] [WSS-0x518000010080] Destroying WebSocket client
[Mon Sep 16 14:02:48 2024] Destroying session 2818990876859785; 0x507000013810
[Mon Sep 16 14:02:48 2024] Detaching handle from Janus SFU plugin; 0x5130000202c0 0x504000010290 0x5130000202c0 0x50200004d490
[Mon Sep 16 14:02:48 2024] Detaching handle from Janus SFU plugin; 0x513000020800 0x504000010550 0x513000020800 0x50200006ee30
[Mon Sep 16 14:02:48 2024] Detaching handle from Janus SFU plugin; 0x51300001ff40 0x504000010090 0x51300001ff40 0x502000038150
[Mon Sep 16 14:02:48 2024] Destroying SFU session 0x504000010290...
[Mon Sep 16 14:02:48 2024] Destroying SFU session 0x504000010550...
[Mon Sep 16 14:02:48 2024] Hanging up WebRTC media on 0x504000010090.
[Mon Sep 16 14:02:48 2024] Sessions:
Session 0x50e0000600d0
Session 0x50e000060ed0
Publishers by Room:
Room forest:
Publisher 0x50e0000600d0
Publishers by User:
User 3103067687: Publisher 0x50e0000600d0
Subscribers by User:
User 3103067687:
Subscriber 0x50e000060ed0
Publisher to Subscribers Mapping:
Forward Mapping:
Publisher 0x50e0000600d0:
Inverse Mapping:
Subscriber 0x50e000060ed0:
Blockers to Miscreants Mapping:
Forward Mapping:
Inverse Mapping:
[Mon Sep 16 14:02:48 2024] [5342214617638961] Handle and related resources freed; 0x513000020800 0x507000013810
[Mon Sep 16 14:02:48 2024] [5353170401350935] WebRTC resources freed; 0x51300001ff40 0x507000013810
[Mon Sep 16 14:02:48 2024] Destroying SFU session 0x504000010090...
[Mon Sep 16 14:02:48 2024] Sessions:
Session 0x50e0000600d0
Publishers by Room:
Room forest:
Publisher 0x50e0000600d0
Publishers by User:
User 3103067687: Publisher 0x50e0000600d0
Subscribers by User:
Publisher to Subscribers Mapping:
Forward Mapping:
Publisher 0x50e0000600d0:
Inverse Mapping:
Blockers to Miscreants Mapping:
Forward Mapping:
Inverse Mapping:
[Mon Sep 16 14:02:48 2024] Sessions:
Publishers by Room:
Publishers by User:
Subscribers by User:
Publisher to Subscribers Mapping:
Forward Mapping:
Inverse Mapping:
Blockers to Miscreants Mapping:
Forward Mapping:
Inverse Mapping:
[Mon Sep 16 14:02:48 2024] [5353170401350935] Handle and related resources freed; 0x51300001ff40 0x507000013810
^CStopping server, please wait...
[Mon Sep 16 14:02:51 2024] Ending sessions timeout watchdog...
[Mon Sep 16 14:02:51 2024] Sessions watchdog stopped
[Mon Sep 16 14:02:51 2024] Closing transport plugins:
[Mon Sep 16 14:02:51 2024] Stopping webserver(s)...
[Mon Sep 16 14:02:51 2024] JANUS REST (HTTP/HTTPS) transport plugin destroyed!
[Mon Sep 16 14:02:51 2024] WebSockets thread ended
[Mon Sep 16 14:02:51 2024] JANUS WebSockets transport plugin destroyed!
[Mon Sep 16 14:02:51 2024] Ending requests thread...
[Mon Sep 16 14:02:51 2024] Leaving Janus requests handler thread
[Mon Sep 16 14:02:51 2024] Destroying sessions...
[Mon Sep 16 14:02:51 2024] Freeing crypto resources...
[Mon Sep 16 14:02:51 2024] De-initializing SCTP...
[Mon Sep 16 14:02:51 2024] Closing plugins:
[Mon Sep 16 14:02:51 2024] Janus SFU plugin destroyed!
[Mon Sep 16 14:02:51 2024] Closing event handlers:
Bye!
=================================================================
==1==ERROR: LeakSanitizer: detected memory leaks
Direct leak of 34920 byte(s) in 1 object(s) allocated from:
#0 0x7d872c9074d0 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x7d872c4767a1 in g_malloc0 (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x637a1) (BuildId: 9753724b85d60f97b5d5663181ef7f4e69a62131)
#2 0x64ea6de6ed9f in janus_sctp_association_create /janus-gateway/sctp.c:189
#3 0x64ea6dd07b31 in janus_dtls_srtp_create_sctp /janus-gateway/dtls.c:688
#4 0x64ea6ddcbaaa in janus_process_incoming_request /janus-gateway/janus.c:1662
#5 0x64ea6dde8459 in janus_transport_task /janus-gateway/janus.c:3478
#6 0x7d872c4a4541 (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x91541) (BuildId: 9753724b85d60f97b5d5663181ef7f4e69a62131)
#7 0x7d872c49ec81 (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x8bc81) (BuildId: 9753724b85d60f97b5d5663181ef7f4e69a62131)
#8 0x7d872c86a109 in asan_thread_start ../../../../src/libsanitizer/asan/asan_interceptors.cpp:234
Indirect leak of 21848 byte(s) in 1 object(s) allocated from:
#0 0x7d872c907b37 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x7d872bf60103 in BUF_MEM_grow (/lib/x86_64-linux-gnu/libcrypto.so.3+0x11b103) (BuildId: c503df82cf13681b2f81e1097e857e3fc50679b1)
#2 0x7d872c3c18e1 (/lib/x86_64-linux-gnu/libssl.so.3+0x698e1) (BuildId: 5308a023a33e723a67a8a09747c70b8b56278446)
#3 0x64ea6dd078d1 in janus_dtls_srtp_handshake /janus-gateway/dtls.c:667
#4 0x64ea6dda1fd8 in janus_ice_outgoing_traffic_handle /janus-gateway/ice.c:4841
#5 0x64ea6dd30fb4 in janus_ice_outgoing_traffic_dispatch /janus-gateway/ice.c:518
#6 0x7d872c4705b4 (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x5d5b4) (BuildId: 9753724b85d60f97b5d5663181ef7f4e69a62131)
#7 0x7d872c4cf716 (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0xbc716) (BuildId: 9753724b85d60f97b5d5663181ef7f4e69a62131)
#8 0x7d872c470f76 in g_main_loop_run (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x5df76) (BuildId: 9753724b85d60f97b5d5663181ef7f4e69a62131)
#9 0x64ea6dd2aa58 in janus_ice_static_event_loop_thread /janus-gateway/ice.c:230
#10 0x7d872c49ec81 (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x8bc81) (BuildId: 9753724b85d60f97b5d5663181ef7f4e69a62131)
#11 0x7d872c86a109 in asan_thread_start ../../../../src/libsanitizer/asan/asan_interceptors.cpp:234in switchboard.rs
impl Switchboard {
pub fn print_state(&self) {
let mut output = String::new();
// Print sessions
output.push_str("Sessions:\n");
for session in &self.sessions {
output.push_str(&format!(" Session {:p}\n", Arc::as_ptr(session)));
}
// Print publishers_by_room
output.push_str("Publishers by Room:\n");
for (room_id, sessions) in &self.publishers_by_room {
output.push_str(&format!(" Room {}:\n", room_id));
for session in sessions {
output.push_str(&format!(" Publisher {:p}\n", Arc::as_ptr(session)));
}
}
// Print publishers_by_user
output.push_str("Publishers by User:\n");
for (user_id, session) in &self.publishers_by_user {
output.push_str(&format!(" User {}: Publisher {:p}\n", user_id, Arc::as_ptr(session)));
}
// Print subscribers_by_user
output.push_str("Subscribers by User:\n");
for (user_id, sessions) in &self.subscribers_by_user {
output.push_str(&format!(" User {}:\n", user_id));
for session in sessions {
output.push_str(&format!(" Subscriber {:p}\n", Arc::as_ptr(session)));
}
}
// Print publisher_to_subscribers
output.push_str("Publisher to Subscribers Mapping:\n");
output.push_str(" Forward Mapping:\n");
for (publisher, subscribers) in &self.publisher_to_subscribers.forward_mapping {
output.push_str(&format!(" Publisher {:p}:\n", Arc::as_ptr(publisher)));
for subscriber in subscribers {
output.push_str(&format!(" Subscriber {:p}\n", Arc::as_ptr(subscriber)));
}
}
output.push_str(" Inverse Mapping:\n");
for (subscriber, publishers) in &self.publisher_to_subscribers.inverse_mapping {
output.push_str(&format!(" Subscriber {:p}:\n", Arc::as_ptr(subscriber)));
for publisher in publishers {
output.push_str(&format!(" Publisher {:p}\n", Arc::as_ptr(publisher)));
}
}
// Print blockers_to_miscreants
output.push_str("Blockers to Miscreants Mapping:\n");
output.push_str(" Forward Mapping:\n");
for (blocker, miscreants) in &self.blockers_to_miscreants.forward_mapping {
output.push_str(&format!(" Blocker {}:\n", blocker));
for miscreant in miscreants {
output.push_str(&format!(" Miscreant {}\n", miscreant));
}
}
output.push_str(" Inverse Mapping:\n");
for (miscreant, blockers) in &self.blockers_to_miscreants.inverse_mapping {
output.push_str(&format!(" Miscreant {}:\n", miscreant));
for blocker in blockers {
output.push_str(&format!(" Blocker {}\n", blocker));
}
}
// Output the state using janus_info!
janus_info!("{}", output);
}and in lib.rs in destroy_session
sess.destroyed.store(true, Ordering::Relaxed);
switchboard.print_state();
See leak stacktrace in https://github.com/networked-aframe/janus-plugin-sfu/pull/11#issuecomment-2105213793 and versions tested in https://github.com/networked-aframe/janus-plugin-sfu/pull/11#issuecomment-2096559942