search

osrg / gobgp

BGP implemented in the Go Programming Language
https://osrg.github.io/gobgp/
Apache License 2.0
3.59k stars 684 forks source link

SIGSEGV when querying EVPN RIB #2352

Open jefftant opened 3 years ago

jefftant commented 3 years ago

Linux ubuntu-xenial 4.4.0-116-generic #140-Ubuntu, 16vCPU's and 32Gb of memory, v2.21.0

{"Key":"192.168.1.1","Reason":"read-failed","State":"BGP_FSM_ESTABLISHED","Topic":"Peer","level":"info","msg":"Peer Down","time":"2020-11-20T17:38:38Z"} fatal error: unexpected signal during runtime execution [signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x806aaa7]

runtime stack: runtime.throw(0x8791b34, 0x2a) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/panic.go:1116 +0x6a runtime.sigpanic() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/signal_unix.go:679 +0x39d runtime.(mheap).allocNeedsZero(0x8d438e0, 0xffb40000, 0x9, 0xba7da460) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/mheap.go:932 +0xc7 runtime.(mheap).allocSpan(0x8d438e0, 0x69, 0x8050000, 0x8d53bc8, 0x935df84) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/mheap.go:1167 +0x10a runtime.(mheap).alloc.func1() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/mheap.go:871 +0x4c runtime.(mheap).alloc(0x8d438e0, 0x69, 0x100, 0x2ee936) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/mheap.go:865 +0x63 runtime.largeAlloc(0xd12a4, 0x80a0001, 0x1) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/malloc.go:1152 +0x76 runtime.mallocgc.func1() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/malloc.go:1047 +0x39 runtime.systemstack(0x96022a0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/asm_386.s:395 +0x53 runtime.mstart() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/proc.go:1056

goroutine 38 [running]: runtime.systemstack_switch() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/asm_386.s:356 fp=0x74b17004 sp=0x74b17000 pc=0x80a6280 runtime.mallocgc(0xd12a4, 0x876b3c0, 0x1, 0x16) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/malloc.go:1046 +0x6aa fp=0x74b17058 sp=0x74b17004 pc=0x8052e6a runtime.makeslice(0x876b3c0, 0x0, 0x344a9, 0x1) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/slice.go:49 +0x4f fp=0x74b1706c sp=0x74b17058 pc=0x808faef github.com/osrg/gobgp/internal/pkg/table.(TableManager).GetPathList(0x93485c0, 0x87724ac, 0x6, 0x0, 0x74b17118, 0x1, 0x1, 0x20, 0x86d3be0, 0x1) /home/runner/work/gobgp/gobgp/internal/pkg/table/table_manager.go:311 +0x5f fp=0x74b170bc sp=0x74b1706c pc=0x859346f github.com/osrg/gobgp/internal/pkg/table.(TableManager).handleMacMobility(0x93485c0, 0x19a932c0, 0x9ab9db20, 0x90e4214, 0x9348e01) /home/runner/work/gobgp/gobgp/internal/pkg/table/table_manager.go:233 +0xee fp=0x74b1715c sp=0x74b170bc pc=0x85928be github.com/osrg/gobgp/internal/pkg/table.(TableManager).Update(0x93485c0, 0x19a932c0, 0x74b17258, 0x1, 0x1) /home/runner/work/gobgp/gobgp/internal/pkg/table/table_manager.go:205 +0xea fp=0x74b171a4 sp=0x74b1715c pc=0x859269a github.com/osrg/gobgp/pkg/server.(BgpServer).propagateUpdate(0x91b6280, 0x90bb9b0, 0xdbc83848, 0x1, 0x1) /home/runner/work/gobgp/gobgp/pkg/server/server.go:1193 +0x1bb fp=0x74b1728c sp=0x74b171a4 pc=0x85ce16b github.com/osrg/gobgp/pkg/server.(BgpServer).handleFSMMessage(0x91b6280, 0x90bb9b0, 0x198656d0) /home/runner/work/gobgp/gobgp/pkg/server/server.go:1585 +0x275f fp=0x74b17da4 sp=0x74b1728c pc=0x85d1f4f github.com/osrg/gobgp/pkg/server.(BgpServer).Serve.func1(0x198656d0) /home/runner/work/gobgp/gobgp/pkg/server/server.go:372 +0x583 fp=0x74b17f4c sp=0x74b17da4 pc=0x85e9733 github.com/osrg/gobgp/pkg/server.(*BgpServer).Serve(0x91b6280) /home/runner/work/gobgp/gobgp/pkg/server/server.go:412 +0x534 fp=0x74b17fe8 sp=0x74b17f4c pc=0x85c8fb4 runtime.goexit() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/asm_386.s:1337 +0x1 fp=0x74b17fec sp=0x74b17fe8 pc=0x80a7ab1 created by main.main /home/runner/work/gobgp/gobgp/cmd/gobgpd/main.go:153 +0x335

goroutine 1 [chan receive, 100 minutes]: main.main() /home/runner/work/gobgp/gobgp/cmd/gobgpd/main.go:192 +0x55d

goroutine 20 [IO wait, 100 minutes]: internal/poll.runtime_pollWait(0xe73d6de8, 0x72, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/netpoll.go:203 +0x4b internal/poll.(pollDesc).wait(0x90b2334, 0x72, 0x0, 0x0, 0x8773366) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:87 +0x37 internal/poll.(pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0x90b2320, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_unix.go:384 +0x19c net.(netFD).accept(0x90b2320, 0x7e11d600, 0x3, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/fd_unix.go:238 +0x27 net.(TCPListener).accept(0x90a2fe0, 0x9322480, 0x904271c, 0x1) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock_posix.go:139 +0x27 net.(TCPListener).AcceptTCP(0x90a2fe0, 0x904271c, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock.go:248 +0x4f github.com/osrg/gobgp/pkg/server.newTCPListener.func1(0x90a2fe0, 0x90947c0, 0x9322480, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/server.go:83 +0x54 created by github.com/osrg/gobgp/pkg/server.newTCPListener /home/runner/work/gobgp/gobgp/pkg/server/server.go:81 +0x1fd

goroutine 33 [chan receive, 100 minutes]: main.init.0.func1() /home/runner/work/gobgp/gobgp/cmd/gobgpd/util.go:37 +0xbc created by main.init.0 /home/runner/work/gobgp/gobgp/cmd/gobgpd/util.go:34 +0x2b

goroutine 34 [syscall, 100 minutes]: os/signal.signal_recv(0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/sigqueue.go:147 +0x12f os/signal.loop() /opt/hostedtoolcache/go/1.14.10/x64/src/os/signal/signal_unix.go:23 +0x1a created by os/signal.Notify.func1 /opt/hostedtoolcache/go/1.14.10/x64/src/os/signal/signal.go:127 +0x33

goroutine 36 [IO wait, 100 minutes]: internal/poll.runtime_pollWait(0xe73d6e7c, 0x72, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/netpoll.go:203 +0x4b internal/poll.(pollDesc).wait(0x92ae014, 0x72, 0x0, 0x0, 0x8773366) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:87 +0x37 internal/poll.(pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0x92ae000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_unix.go:384 +0x19c net.(netFD).accept(0x92ae000, 0x83dc9e01, 0x90465a0, 0xf773a088) /opt/hostedtoolcache/go/1.14.10/x64/src/net/fd_unix.go:238 +0x27 net.(TCPListener).accept(0x900c150, 0xffffffff, 0x80530bc, 0x20) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock_posix.go:139 +0x27 net.(TCPListener).Accept(0x900c150, 0x8729fa0, 0x9024580, 0x86a3de0, 0x8d36a30) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock.go:261 +0x57 net/http.(Server).Serve(0x9078000, 0x8899780, 0x900c150, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/http/server.go:2930 +0x1eb net/http.(Server).ListenAndServe(0x9078000, 0x9078000, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/http/server.go:2859 +0xa0 net/http.ListenAndServe(...) /opt/hostedtoolcache/go/1.14.10/x64/src/net/http/server.go:3115 main.main.func1(0x91cb860) /home/runner/work/gobgp/gobgp/cmd/gobgpd/main.go:89 +0x6a created by main.main /home/runner/work/gobgp/gobgp/cmd/gobgpd/main.go:88 +0x15f3

goroutine 37 [semacquire, 100 minutes]: sync.runtime_Semacquire(0x9116018) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/sema.go:56 +0x36 sync.(WaitGroup).Wait(0x9116010) /opt/hostedtoolcache/go/1.14.10/x64/src/sync/waitgroup.go:130 +0x7c github.com/osrg/gobgp/pkg/server.(server).serve(0x91846d0, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/grpc_server.go:100 +0x45f github.com/osrg/gobgp/pkg/server.NewBgpServer.func1(0x91846d0) /home/runner/work/gobgp/gobgp/pkg/server/server.go:163 +0x21 created by github.com/osrg/gobgp/pkg/server.NewBgpServer /home/runner/work/gobgp/gobgp/pkg/server/server.go:162 +0x409

goroutine 49 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0xe73d6f10, 0x72, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/netpoll.go:203 +0x4b internal/poll.(pollDesc).wait(0x9124014, 0x72, 0x0, 0x0, 0x8773366) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:87 +0x37 internal/poll.(pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0x9124000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_unix.go:384 +0x19c net.(netFD).accept(0x9124000, 0x7e11d600, 0x3, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/fd_unix.go:238 +0x27 net.(TCPListener).accept(0x9110010, 0x9120e74, 0x9120e78, 0xc) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock_posix.go:139 +0x27 net.(TCPListener).Accept(0x9110010, 0x87a6db8, 0x91f92c0, 0x889fbe0, 0x416c3f28) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock.go:261 +0x57 google.golang.org/grpc.(Server).Serve(0x91f92c0, 0x8899780, 0x9110010, 0x0, 0x0) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.5.1/server.go:444 +0x14f github.com/osrg/gobgp/pkg/server.(server).serve.func1(0x8899780, 0x9110010) /home/runner/work/gobgp/gobgp/pkg/server/grpc_server.go:89 +0x78 created by github.com/osrg/gobgp/pkg/server.(*server).serve /home/runner/work/gobgp/gobgp/pkg/server/grpc_server.go:98 +0x440

goroutine 21 [IO wait, 100 minutes]: internal/poll.runtime_pollWait(0xe73d6d54, 0x72, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/netpoll.go:203 +0x4b internal/poll.(pollDesc).wait(0x90b2384, 0x72, 0x0, 0x0, 0x8773366) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:87 +0x37 internal/poll.(pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0x90b2370, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_unix.go:384 +0x19c net.(netFD).accept(0x90b2370, 0x85ec37d, 0x9322440, 0xc) /opt/hostedtoolcache/go/1.14.10/x64/src/net/fd_unix.go:238 +0x27 net.(TCPListener).accept(0x90a2ff0, 0x20, 0x90ba1ac, 0x90ba1a4) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock_posix.go:139 +0x27 net.(TCPListener).AcceptTCP(0x90a2ff0, 0x90ca758, 0x90ba180, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock.go:248 +0x4f github.com/osrg/gobgp/pkg/server.newTCPListener.func1(0x90a2ff0, 0x9094880, 0x9322480, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/server.go:83 +0x54 created by github.com/osrg/gobgp/pkg/server.newTCPListener /home/runner/work/gobgp/gobgp/pkg/server/server.go:81 +0x1fd

goroutine 1763 [select]: github.com/eapache/channels.(*InfiniteChannel).infiniteBuffer(0x7c577a30) /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:49 +0xe4 created by github.com/eapache/channels.NewInfiniteChannel /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:19 +0x126

goroutine 23 [select]: github.com/eapache/channels.(*InfiniteChannel).infiniteBuffer(0x90a3230) /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:49 +0xe4 created by github.com/eapache/channels.NewInfiniteChannel /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:19 +0x126

goroutine 148 [select, 100 minutes]: github.com/eapache/channels.(*InfiniteChannel).infiniteBuffer(0x9184a40) /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:49 +0xe4 created by github.com/eapache/channels.NewInfiniteChannel /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:19 +0x126

goroutine 26 [runnable]: github.com/eapache/channels.(*InfiniteChannel).infiniteBuffer(0x90a3420) /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:49 +0xe4 created by github.com/eapache/channels.NewInfiniteChannel /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:19 +0x126

goroutine 149 [select]: github.com/osrg/gobgp/pkg/server.(fsmHandler).established(0x9258300, 0x889a560, 0x925a020, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1757 +0x32b github.com/osrg/gobgp/pkg/server.(fsmHandler).loop(0x9258300, 0x889a560, 0x925a020, 0x9190ea0, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1870 +0x851 created by github.com/osrg/gobgp/pkg/server.newFSMHandler /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:406 +0x163

goroutine 151 [IO wait]: internal/poll.runtime_pollWait(0xe73d6c2c, 0x72, 0xffffffff) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/netpoll.go:203 +0x4b internal/poll.(pollDesc).wait(0x91ac834, 0x72, 0x0, 0x13, 0xffffffff) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:87 +0x37 internal/poll.(pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0x91ac820, 0x6bb36060, 0x13, 0x13, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_unix.go:169 +0x168 net.(netFD).Read(0x91ac820, 0x6bb36060, 0x13, 0x13, 0x8890ea0, 0x7ff, 0x1) /opt/hostedtoolcache/go/1.14.10/x64/src/net/fd_unix.go:202 +0x3f net.(conn).Read(0x9186e10, 0x6bb36060, 0x13, 0x13, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/net.go:184 +0x67 io.ReadAtLeast(0x8890ea0, 0x9186e10, 0x6bb36060, 0x13, 0x13, 0x13, 0x86440b6f, 0x90d2500, 0x85ac0c9) /opt/hostedtoolcache/go/1.14.10/x64/src/io/io.go:310 +0x74 io.ReadFull(...) /opt/hostedtoolcache/go/1.14.10/x64/src/io/io.go:329 github.com/osrg/gobgp/pkg/server.readAll(0x889fbe0, 0x9186e10, 0x13, 0x0, 0x66883868, 0x1, 0x1, 0x975481e0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:796 +0x85 github.com/osrg/gobgp/pkg/server.(fsmHandler).recvMessageWithError(0x9258300, 0x0, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:933 +0x6e github.com/osrg/gobgp/pkg/server.(fsmHandler).recvMessageloop(0x9258300, 0x889a560, 0x925a020, 0x9190f20, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1721 +0x5e created by github.com/osrg/gobgp/pkg/server.(fsmHandler).established /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1743 +0x175

goroutine 150 [select]: github.com/osrg/gobgp/pkg/server.(fsmHandler).sendMessageloop(0x9258300, 0x889a560, 0x925a020, 0x9190f20, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1683 +0x201 created by github.com/osrg/gobgp/pkg/server.(fsmHandler).established /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1741 +0x122

goroutine 1753 [select]: github.com/osrg/gobgp/pkg/server.(fsmHandler).connectLoop(0x1947c740, 0x889a560, 0x9aea23e0, 0x9aa17b90) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:526 +0x335 created by github.com/osrg/gobgp/pkg/server.(fsmHandler).active /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:602 +0x1080

goroutine 1764 [select]: github.com/osrg/gobgp/pkg/server.(fsmHandler).active(0x1947c740, 0x889a560, 0x9acbec40, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:611 +0x25f github.com/osrg/gobgp/pkg/server.(fsmHandler).loop(0x1947c740, 0x889a560, 0x9acbec40, 0x9a8f2940, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1864 +0x7da created by github.com/osrg/gobgp/pkg/server.newFSMHandler /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:406 +0x163

serejkus commented 3 years ago

Does the host/container has enough memory? Go runtime requested abot 400 kilobytes of RAM:

 /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/mheap.go:871 +0x4c
runtime.(*mheap).alloc(0x8d438e0, 0x69, 0x100, 0x2ee936)

The exact line with failure does not show that it was OOM: https://github.com/golang/go/blob/go1.14.10/src/runtime/mheap.go#L932 . I don't see any relevant fixes in 1.14.10+ releases.

Upcoming go release 1.16 will change interaction between go runtime allocator and the OS, maybe it'll help. Go 1.15 had some allocator tunings too, and it is more stable. It might be worth trying to rebuild gobgpd with more fresh go compiler.