Open anacrolix opened 2 years ago
Running confluence with some implicit websocket trackers results in a slow leak of UDP sockets. Here's some relevant stack traces:
745 @ 0x44d476 0x45cf12 0x938e45 0x47d121 # 0x938e44 github.com/pion/ice/v2.(*candidateBase).recvLoop+0xc4 /root/go/pkg/mod/github.com/pion/ice/v2@v2.2.6/candidate_base.go:213 186 @ 0x44d476 0x4197ec 0x419258 0x9305c8 0x47d121 # 0x9305c7 github.com/pion/ice/v2.(*Agent).startOnConnectionStateChangeRoutine.func1+0x47 /root/go/pkg/mod/github.com/pion/ice/v2@v2.2.6/agent.go:413 186 @ 0x44d476 0x45cf12 0x52147e 0x47d121 # 0x52147d context.propagateCancel.func1+0x9d /root/src/go.master/src/context/context.go:279 186 @ 0x44d476 0x45cf12 0x92ecc5 0x47d121 # 0x92ecc4 github.com/pion/ice/v2.(*Agent).taskLoop+0x144 /root/go/pkg/mod/github.com/pion/ice/v2@v2.2.6/agent.go:223 186 @ 0x44d476 0x45cf12 0x93032d 0x47d121 # 0x93032c github.com/pion/ice/v2.(*Agent).startOnConnectionStateChangeRoutine.func2+0xac /root/go/pkg/mod/github.com/pion/ice/v2@v2.2.6/agent.go:422 164 @ 0x44d476 0x4465f3 0x477949 0x4dc6b2 0x4e2e05 0x4e2de9 0x573805 0x8fa482 0x90400e 0x903ff4 0x908ea7 0x47d121 # 0x477948 internal/poll.runtime_pollWait+0x88 /root/src/go.master/src/runtime/netpoll.go:233 # 0x4dc6b1 internal/poll.(*pollDesc).wait+0x31 /root/src/go.master/src/internal/poll/fd_poll_runtime.go:83 # 0x4e2e04 internal/poll.(*pollDesc).waitRead+0x144 /root/src/go.master/src/internal/poll/fd_poll_runtime.go:88 # 0x4e2de8 internal/poll.(*FD).RawRead+0x128 /root/src/go.master/src/internal/poll/fd_unix.go:766 # 0x573804 net.(*rawConn).Read+0x44 /root/src/go.master/src/net/rawconn.go:43 # 0x8fa481 golang.org/x/net/internal/socket.(*Conn).recvMsg+0x201 /root/go/pkg/mod/golang.org/x/net@v0.0.0-20220630215102-69896b714898/internal/socket/rawconn_msg.go:29 # 0x90400d golang.org/x/net/internal/socket.(*Conn).RecvMsg+0x1cd /root/go/pkg/mod/golang.org/x/net@v0.0.0-20220630215102-69896b714898/internal/socket/socket.go:247 # 0x903ff3 golang.org/x/net/ipv4.(*payloadHandler).ReadFrom+0x1b3 /root/go/pkg/mod/golang.org/x/net@v0.0.0-20220630215102-69896b714898/ipv4/payload_cmsg.go:32 # 0x908ea6 github.com/pion/mdns.(*Conn).start+0x106 /root/go/pkg/mod/github.com/pion/mdns@v0.0.5/conn.go:258 110 @ 0x44d476 0x45cf12 0x8b9bc8 0x94f405 0x8e92fa 0x47d121 # 0x8b9bc7 github.com/pion/transport/packetio.(*Buffer).Read+0x167 /root/go/pkg/mod/github.com/pion/transport@v0.13.1/packetio/buffer.go:266 # 0x94f404 github.com/pion/webrtc/v3/internal/mux.(*Endpoint).Read+0x24 /root/go/pkg/mod/github.com/pion/webrtc/v3@v3.1.42/internal/mux/endpoint.go:37 # 0x8e92f9 github.com/pion/srtp/v2.(*session).start.func1+0xb9 /root/go/pkg/mod/github.com/pion/srtp/v2@v2.0.9/session.go:134 56 @ 0x44d476 0x45cf12 0x930f2b 0x47d121 # 0x930f2a github.com/pion/ice/v2.(*Agent).connectivityChecks+0x1aa /root/go/pkg/mod/github.com/pion/ice/v2@v2.2.6/agent.go:539 55 @ 0x44d476 0x4197ec 0x419258 0x7c4cc6 0x47d121 # 0x7c4cc5 github.com/anacrolix/torrent/peer_protocol.handshakeWriter+0x65 /root/dev-gopath/src/github.com/anacrolix/torrent/peer_protocol/handshake.go:23 55 @ 0x44d476 0x4197ec 0x419258 0x7fcea5 0x7fce88 0x989d08 0x47d121 # 0x7fcea4 github.com/pion/sctp.(*Association).AcceptStream+0x44 /root/go/pkg/mod/github.com/pion/sctp@v1.8.2/association.go:1307 # 0x7fce87 github.com/pion/datachannel.Accept+0x27 /root/go/pkg/mod/github.com/pion/datachannel@v1.5.2/datachannel.go:116 # 0x989d07 github.com/pion/webrtc/v3.(*SCTPTransport).acceptDataChannels+0x287 /root/go/pkg/mod/github.com/pion/webrtc/v3@v3.1.42/sctptransport.go:175 55 @ 0x44d476 0x4197ec 0x419258 0x8e9bec 0x977445 0x47d121 # 0x8e9beb github.com/pion/srtp/v2.(*SessionSRTCP).AcceptStream+0x2b /root/go/pkg/mod/github.com/pion/srtp/v2@v2.0.9/session_srtcp.go:92 # 0x977444 github.com/pion/webrtc/v3.(*PeerConnection).undeclaredMediaProcessor.func2+0xe4 /root/go/pkg/mod/github.com/pion/webrtc/v3@v3.1.42/peerconnection.go:1603 55 @ 0x44d476 0x4197ec 0x419258 0x8eabcc 0x9776b9 0x47d121 # 0x8eabcb github.com/pion/srtp/v2.(*SessionSRTP).AcceptStream+0x2b /root/go/pkg/mod/github.com/pion/srtp/v2@v2.0.9/session_srtp.go:94 # 0x9776b8 github.com/pion/webrtc/v3.(*PeerConnection).undeclaredMediaProcessor.func1+0xd8 /root/go/pkg/mod/github.com/pion/webrtc/v3@v3.1.42/peerconnection.go:1565 55 @ 0x44d476 0x45cf12 0x7dc4a6 0x47d121 # 0x7dc4a5 github.com/pion/sctp.(*Association).writeLoop+0x205 /root/go/pkg/mod/github.com/pion/sctp@v1.8.2/association.go:558 55 @ 0x44d476 0x45cf12 0x8b7489 0x47d121 # 0x8b7488 github.com/pion/transport/connctx.(*connCtx).ReadContext.func1+0xc8 /root/go/pkg/mod/github.com/pion/transport@v0.13.1/connctx/connctx.go:78 55 @ 0x44d476 0x45cf12 0x8b9bc8 0x948973 0x94fe0b 0x47d121 # 0x8b9bc7 github.com/pion/transport/packetio.(*Buffer).Read+0x167 /root/go/pkg/mod/github.com/pion/transport@v0.13.1/packetio/buffer.go:266 # 0x948972 github.com/pion/ice/v2.(*Conn).Read+0x72 /root/go/pkg/mod/github.com/pion/ice/v2@v2.2.6/transport.go:73 # 0x94fe0a github.com/pion/webrtc/v3/internal/mux.(*Mux).readLoop+0xca /root/go/pkg/mod/github.com/pion/webrtc/v3@v3.1.42/internal/mux/mux.go:107 55 @ 0x44d476 0x45cf12 0x8b9bc8 0x94f405 0x8b71ce 0x8bfc0e 0x8c24c5 0x47d121 # 0x8b9bc7 github.com/pion/transport/packetio.(*Buffer).Read+0x167 /root/go/pkg/mod/github.com/pion/transport@v0.13.1/packetio/buffer.go:266 # 0x94f404 github.com/pion/webrtc/v3/internal/mux.(*Endpoint).Read+0x24 /root/go/pkg/mod/github.com/pion/webrtc/v3@v3.1.42/internal/mux/endpoint.go:37 # 0x8b71cd github.com/pion/transport/connctx.(*connCtx).ReadContext+0x22d /root/go/pkg/mod/github.com/pion/transport@v0.13.1/connctx/connctx.go:93 # 0x8bfc0d github.com/pion/dtls/v2.(*Conn).readAndBuffer+0x10d /root/go/pkg/mod/github.com/pion/dtls/v2@v2.1.5/conn.go:570 # 0x8c24c4 github.com/pion/dtls/v2.(*Conn).handshake.func3+0x104 /root/go/pkg/mod/github.com/pion/dtls/v2@v2.1.5/conn.go:852 55 @ 0x44d476 0x45cf12 0x8bdba5 0x7dba7e 0x47d121 # 0x8bdba4 github.com/pion/dtls/v2.(*Conn).Read+0x164 /root/go/pkg/mod/github.com/pion/dtls/v2@v2.1.5/conn.go:294 # 0x7dba7d github.com/pion/sctp.(*Association).readLoop+0x15d /root/go/pkg/mod/github.com/pion/sctp@v1.8.2/association.go:509 55 @ 0x44d476 0x45cf12 0x8d42d2 0x8d313d 0x8c2ae5 0x47d121 # 0x8d42d1 github.com/pion/dtls/v2.(*handshakeFSM).finish+0x251 /root/go/pkg/mod/github.com/pion/dtls/v2@v2.1.5/handshaker.go:312 # 0x8d313c github.com/pion/dtls/v2.(*handshakeFSM).Run+0x43c /root/go/pkg/mod/github.com/pion/dtls/v2@v2.1.5/handshaker.go:179 # 0x8c2ae4 github.com/pion/dtls/v2.(*Conn).handshake.func2+0x84 /root/go/pkg/mod/github.com/pion/dtls/v2@v2.1.5/conn.go:833 55 @ 0x44d476 0x47927d 0x47925d 0x49146c 0x7f776b 0x7fd425 0x7fd37e 0x9f5dff 0xa0ac46 0x4bad7a 0x7c544d 0x7c541a 0x9ee448 0x9ed925 0x9ec925 0xa1d34a 0x47d121 # 0x47925c sync.runtime_notifyListWait+0x11c /root/src/go.master/src/runtime/sema.go:513 # 0x49146b sync.(*Cond).Wait+0x8b /root/src/go.master/src/sync/cond.go:56 # 0x7f776a github.com/pion/sctp.(*Stream).ReadSCTP+0xaa /root/go/pkg/mod/github.com/pion/sctp@v1.8.2/stream.go:108 # 0x7fd424 github.com/pion/datachannel.(*DataChannel).ReadDataChannel+0x64 /root/go/pkg/mod/github.com/pion/datachannel@v1.5.2/datachannel.go:186 # 0x7fd37d github.com/pion/datachannel.(*DataChannel).Read+0x1d /root/go/pkg/mod/github.com/pion/datachannel@v1.5.2/datachannel.go:179 # 0x9f5dfe github.com/anacrolix/torrent.connStatsReadWriter.Read+0x3e /root/dev-gopath/src/github.com/anacrolix/torrent/conn_stats.go:114 # 0xa0ac45 github.com/anacrolix/torrent.(*rateLimitedReader).Read+0xe5 /root/dev-gopath/src/github.com/anacrolix/torrent/ratelimitreader.go:43 # 0x4bad79 io.ReadAtLeast+0x99 /root/src/go.master/src/io/io.go:331 # 0x7c544c io.ReadFull+0x32c /root/src/go.master/src/io/io.go:350 # 0x7c5419 github.com/anacrolix/torrent/peer_protocol.Handshake+0x2f9 /root/dev-gopath/src/github.com/anacrolix/torrent/peer_protocol/handshake.go:119 # 0x9ee447 github.com/anacrolix/torrent.(*Client).connBtHandshake+0x127 /root/dev-gopath/src/github.com/anacrolix/torrent/client.go:894 # 0x9ed924 github.com/anacrolix/torrent.(*Client).initiateHandshakes+0x244 /root/dev-gopath/src/github.com/anacrolix/torrent/client.go:808 # 0x9ec924 github.com/anacrolix/torrent.(*Client).initiateProtocolHandshakes+0x184 /root/dev-gopath/src/github.com/anacrolix/torrent/client.go:708 # 0xa1d349 github.com/anacrolix/torrent.(*Torrent).onWebRtcConn+0x389 /root/dev-gopath/src/github.com/anacrolix/torrent/torrent.go:1577 39 @ 0x44d476 0x4465f3 0x477949 0x4dc6b2 0x4de365 0x4de352 0x55b3a9 0x57c6f8 0x57a431 0x57a5f0 0x938f29 0x47d121 # 0x477948 internal/poll.runtime_pollWait+0x88 /root/src/go.master/src/runtime/netpoll.go:233 # 0x4dc6b1 internal/poll.(*pollDesc).wait+0x31 /root/src/go.master/src/internal/poll/fd_poll_runtime.go:83 # 0x4de364 internal/poll.(*pollDesc).waitRead+0x1e4 /root/src/go.master/src/internal/poll/fd_poll_runtime.go:88 # 0x4de351 internal/poll.(*FD).ReadFromInet4+0x1d1 /root/src/go.master/src/internal/poll/fd_unix.go:250 # 0x55b3a8 net.(*netFD).readFromInet4+0x28 /root/src/go.master/src/net/fd_posix.go:66 # 0x57c6f7 net.(*UDPConn).readFrom+0x1b7 /root/src/go.master/src/net/udpsock_posix.go:52 # 0x57a430 net.(*UDPConn).readFromUDP+0x30 /root/src/go.master/src/net/udpsock.go:149 # 0x57a5ef net.(*UDPConn).ReadFrom+0x4f /root/src/go.master/src/net/udpsock.go:158 # 0x938f28 github.com/pion/ice/v2.(*candidateBase).recvLoop+0x1a8 /root/go/pkg/mod/github.com/pion/ice/v2@v2.2.6/candidate_base.go:222 36 @ 0x44d476 0x47927d 0x47925d 0x49146c 0x9ded73 0x9f5dff 0xa0ac46 0x4bad7a 0x9f721b 0x9f71eb 0x9eddd6 0x9ee7a5 0x9eb85c 0x47d121 # 0x47925c sync.runtime_notifyListWait+0x11c /root/src/go.master/src/runtime/sema.go:513 # 0x49146b sync.(*Cond).Wait+0x8b /root/src/go.master/src/sync/cond.go:56 # 0x9ded72 github.com/anacrolix/go-libutp.(*Conn).Read+0x92 /root/go/pkg/mod/github.com/anacrolix/go-libutp@v1.2.0/conn.go:151 # 0x9f5dfe github.com/anacrolix/torrent.connStatsReadWriter.Read+0x3e /root/dev-gopath/src/github.com/anacrolix/torrent/conn_stats.go:114 # 0xa0ac45 github.com/anacrolix/torrent.(*rateLimitedReader).Read+0xe5 /root/dev-gopath/src/github.com/anacrolix/torrent/ratelimitreader.go:43 # 0x4bad79 io.ReadAtLeast+0x99 /root/src/go.master/src/io/io.go:331 # 0x9f721a io.ReadFull+0x9a /root/src/go.master/src/io/io.go:350 # 0x9f71ea github.com/anacrolix/torrent.handleEncryption+0x6a /root/dev-gopath/src/github.com/anacrolix/torrent/handshake.go:45 # 0x9eddd5 github.com/anacrolix/torrent.(*Client).receiveHandshakes+0x195 /root/dev-gopath/src/github.com/anacrolix/torrent/client.go:854 # 0x9ee7a4 github.com/anacrolix/torrent.(*Client).runReceivedConn+0xc4 /root/dev-gopath/src/github.com/anacrolix/torrent/client.go:914 # 0x9eb85b github.com/anacrolix/torrent.(*Client).incomingConnection+0x21b /root/dev-gopath/src/github.com/anacrolix/torrent/client.go:562 30 @ 0x44d476 0x4465f3 0x477949 0x4dc6b2 0x4de6c5 0x4de6b2 0x55b4c9 0x57c5c5 0x57a431 0x57a5f0 0x938f29 0x47d121 # 0x477948 internal/poll.runtime_pollWait+0x88 /root/src/go.master/src/runtime/netpoll.go:233 # 0x4dc6b1 internal/poll.(*pollDesc).wait+0x31 /root/src/go.master/src/internal/poll/fd_poll_runtime.go:83 # 0x4de6c4 internal/poll.(*pollDesc).waitRead+0x1e4 /root/src/go.master/src/internal/poll/fd_poll_runtime.go:88 # 0x4de6b1 internal/poll.(*FD).ReadFromInet6+0x1d1 /root/src/go.master/src/internal/poll/fd_unix.go:277 # 0x55b4c8 net.(*netFD).readFromInet6+0x28 /root/src/go.master/src/net/fd_posix.go:72 # 0x57c5c4 net.(*UDPConn).readFrom+0x84 /root/src/go.master/src/net/udpsock_posix.go:59 # 0x57a430 net.(*UDPConn).readFromUDP+0x30 /root/src/go.master/src/net/udpsock.go:149 # 0x57a5ef net.(*UDPConn).ReadFrom+0x4f /root/src/go.master/src/net/udpsock.go:158 # 0x938f28 github.com/pion/ice/v2.(*candidateBase).recvLoop+0x1a8 /root/go/pkg/mod/github.com/pion/ice/v2@v2.2.6/candidate_base.go:222 30 @ 0x44d476 0x45cf12 0x907e31 0x932545 0x47d121 # 0x907e30 github.com/pion/mdns.(*Conn).Query+0x390 /root/go/pkg/mod/github.com/pion/mdns@v0.0.5/conn.go:139 # 0x932544 github.com/pion/ice/v2.(*Agent).resolveAndAddMulticastCandidate+0x64 /root/go/pkg/mod/github.com/pion/ice/v2@v2.2.6/agent.go:761
It may be related to https://github.com/anacrolix/torrent/issues/703, but I'm leaning away from that. I need to run an inuse_space check when this problem shows up.
Running confluence with some implicit websocket trackers results in a slow leak of UDP sockets. Here's some relevant stack traces:
It may be related to https://github.com/anacrolix/torrent/issues/703, but I'm leaning away from that. I need to run an inuse_space check when this problem shows up.