Open jefftant opened 3 years ago
Does the host/container has enough memory? Go runtime requested abot 400 kilobytes of RAM:
/opt/hostedtoolcache/go/1.14.10/x64/src/runtime/mheap.go:871 +0x4c
runtime.(*mheap).alloc(0x8d438e0, 0x69, 0x100, 0x2ee936)
The exact line with failure does not show that it was OOM: https://github.com/golang/go/blob/go1.14.10/src/runtime/mheap.go#L932 . I don't see any relevant fixes in 1.14.10+ releases.
Upcoming go release 1.16 will change interaction between go runtime allocator and the OS, maybe it'll help. Go 1.15 had some allocator tunings too, and it is more stable. It might be worth trying to rebuild gobgpd with more fresh go compiler.
Linux ubuntu-xenial 4.4.0-116-generic #140-Ubuntu, 16vCPU's and 32Gb of memory, v2.21.0
{"Key":"192.168.1.1","Reason":"read-failed","State":"BGP_FSM_ESTABLISHED","Topic":"Peer","level":"info","msg":"Peer Down","time":"2020-11-20T17:38:38Z"} fatal error: unexpected signal during runtime execution [signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x806aaa7]
runtime stack: runtime.throw(0x8791b34, 0x2a) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/panic.go:1116 +0x6a runtime.sigpanic() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/signal_unix.go:679 +0x39d runtime.(mheap).allocNeedsZero(0x8d438e0, 0xffb40000, 0x9, 0xba7da460) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/mheap.go:932 +0xc7 runtime.(mheap).allocSpan(0x8d438e0, 0x69, 0x8050000, 0x8d53bc8, 0x935df84) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/mheap.go:1167 +0x10a runtime.(mheap).alloc.func1() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/mheap.go:871 +0x4c runtime.(mheap).alloc(0x8d438e0, 0x69, 0x100, 0x2ee936) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/mheap.go:865 +0x63 runtime.largeAlloc(0xd12a4, 0x80a0001, 0x1) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/malloc.go:1152 +0x76 runtime.mallocgc.func1() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/malloc.go:1047 +0x39 runtime.systemstack(0x96022a0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/asm_386.s:395 +0x53 runtime.mstart() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/proc.go:1056
goroutine 38 [running]: runtime.systemstack_switch() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/asm_386.s:356 fp=0x74b17004 sp=0x74b17000 pc=0x80a6280 runtime.mallocgc(0xd12a4, 0x876b3c0, 0x1, 0x16) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/malloc.go:1046 +0x6aa fp=0x74b17058 sp=0x74b17004 pc=0x8052e6a runtime.makeslice(0x876b3c0, 0x0, 0x344a9, 0x1) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/slice.go:49 +0x4f fp=0x74b1706c sp=0x74b17058 pc=0x808faef github.com/osrg/gobgp/internal/pkg/table.(TableManager).GetPathList(0x93485c0, 0x87724ac, 0x6, 0x0, 0x74b17118, 0x1, 0x1, 0x20, 0x86d3be0, 0x1) /home/runner/work/gobgp/gobgp/internal/pkg/table/table_manager.go:311 +0x5f fp=0x74b170bc sp=0x74b1706c pc=0x859346f github.com/osrg/gobgp/internal/pkg/table.(TableManager).handleMacMobility(0x93485c0, 0x19a932c0, 0x9ab9db20, 0x90e4214, 0x9348e01) /home/runner/work/gobgp/gobgp/internal/pkg/table/table_manager.go:233 +0xee fp=0x74b1715c sp=0x74b170bc pc=0x85928be github.com/osrg/gobgp/internal/pkg/table.(TableManager).Update(0x93485c0, 0x19a932c0, 0x74b17258, 0x1, 0x1) /home/runner/work/gobgp/gobgp/internal/pkg/table/table_manager.go:205 +0xea fp=0x74b171a4 sp=0x74b1715c pc=0x859269a github.com/osrg/gobgp/pkg/server.(BgpServer).propagateUpdate(0x91b6280, 0x90bb9b0, 0xdbc83848, 0x1, 0x1) /home/runner/work/gobgp/gobgp/pkg/server/server.go:1193 +0x1bb fp=0x74b1728c sp=0x74b171a4 pc=0x85ce16b github.com/osrg/gobgp/pkg/server.(BgpServer).handleFSMMessage(0x91b6280, 0x90bb9b0, 0x198656d0) /home/runner/work/gobgp/gobgp/pkg/server/server.go:1585 +0x275f fp=0x74b17da4 sp=0x74b1728c pc=0x85d1f4f github.com/osrg/gobgp/pkg/server.(BgpServer).Serve.func1(0x198656d0) /home/runner/work/gobgp/gobgp/pkg/server/server.go:372 +0x583 fp=0x74b17f4c sp=0x74b17da4 pc=0x85e9733 github.com/osrg/gobgp/pkg/server.(*BgpServer).Serve(0x91b6280) /home/runner/work/gobgp/gobgp/pkg/server/server.go:412 +0x534 fp=0x74b17fe8 sp=0x74b17f4c pc=0x85c8fb4 runtime.goexit() /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/asm_386.s:1337 +0x1 fp=0x74b17fec sp=0x74b17fe8 pc=0x80a7ab1 created by main.main /home/runner/work/gobgp/gobgp/cmd/gobgpd/main.go:153 +0x335
goroutine 1 [chan receive, 100 minutes]: main.main() /home/runner/work/gobgp/gobgp/cmd/gobgpd/main.go:192 +0x55d
goroutine 20 [IO wait, 100 minutes]: internal/poll.runtime_pollWait(0xe73d6de8, 0x72, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/netpoll.go:203 +0x4b internal/poll.(pollDesc).wait(0x90b2334, 0x72, 0x0, 0x0, 0x8773366) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:87 +0x37 internal/poll.(pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0x90b2320, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_unix.go:384 +0x19c net.(netFD).accept(0x90b2320, 0x7e11d600, 0x3, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/fd_unix.go:238 +0x27 net.(TCPListener).accept(0x90a2fe0, 0x9322480, 0x904271c, 0x1) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock_posix.go:139 +0x27 net.(TCPListener).AcceptTCP(0x90a2fe0, 0x904271c, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock.go:248 +0x4f github.com/osrg/gobgp/pkg/server.newTCPListener.func1(0x90a2fe0, 0x90947c0, 0x9322480, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/server.go:83 +0x54 created by github.com/osrg/gobgp/pkg/server.newTCPListener /home/runner/work/gobgp/gobgp/pkg/server/server.go:81 +0x1fd
goroutine 33 [chan receive, 100 minutes]: main.init.0.func1() /home/runner/work/gobgp/gobgp/cmd/gobgpd/util.go:37 +0xbc created by main.init.0 /home/runner/work/gobgp/gobgp/cmd/gobgpd/util.go:34 +0x2b
goroutine 34 [syscall, 100 minutes]: os/signal.signal_recv(0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/sigqueue.go:147 +0x12f os/signal.loop() /opt/hostedtoolcache/go/1.14.10/x64/src/os/signal/signal_unix.go:23 +0x1a created by os/signal.Notify.func1 /opt/hostedtoolcache/go/1.14.10/x64/src/os/signal/signal.go:127 +0x33
goroutine 36 [IO wait, 100 minutes]: internal/poll.runtime_pollWait(0xe73d6e7c, 0x72, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/netpoll.go:203 +0x4b internal/poll.(pollDesc).wait(0x92ae014, 0x72, 0x0, 0x0, 0x8773366) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:87 +0x37 internal/poll.(pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0x92ae000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_unix.go:384 +0x19c net.(netFD).accept(0x92ae000, 0x83dc9e01, 0x90465a0, 0xf773a088) /opt/hostedtoolcache/go/1.14.10/x64/src/net/fd_unix.go:238 +0x27 net.(TCPListener).accept(0x900c150, 0xffffffff, 0x80530bc, 0x20) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock_posix.go:139 +0x27 net.(TCPListener).Accept(0x900c150, 0x8729fa0, 0x9024580, 0x86a3de0, 0x8d36a30) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock.go:261 +0x57 net/http.(Server).Serve(0x9078000, 0x8899780, 0x900c150, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/http/server.go:2930 +0x1eb net/http.(Server).ListenAndServe(0x9078000, 0x9078000, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/http/server.go:2859 +0xa0 net/http.ListenAndServe(...) /opt/hostedtoolcache/go/1.14.10/x64/src/net/http/server.go:3115 main.main.func1(0x91cb860) /home/runner/work/gobgp/gobgp/cmd/gobgpd/main.go:89 +0x6a created by main.main /home/runner/work/gobgp/gobgp/cmd/gobgpd/main.go:88 +0x15f3
goroutine 37 [semacquire, 100 minutes]: sync.runtime_Semacquire(0x9116018) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/sema.go:56 +0x36 sync.(WaitGroup).Wait(0x9116010) /opt/hostedtoolcache/go/1.14.10/x64/src/sync/waitgroup.go:130 +0x7c github.com/osrg/gobgp/pkg/server.(server).serve(0x91846d0, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/grpc_server.go:100 +0x45f github.com/osrg/gobgp/pkg/server.NewBgpServer.func1(0x91846d0) /home/runner/work/gobgp/gobgp/pkg/server/server.go:163 +0x21 created by github.com/osrg/gobgp/pkg/server.NewBgpServer /home/runner/work/gobgp/gobgp/pkg/server/server.go:162 +0x409
goroutine 49 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0xe73d6f10, 0x72, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/netpoll.go:203 +0x4b internal/poll.(pollDesc).wait(0x9124014, 0x72, 0x0, 0x0, 0x8773366) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:87 +0x37 internal/poll.(pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0x9124000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_unix.go:384 +0x19c net.(netFD).accept(0x9124000, 0x7e11d600, 0x3, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/fd_unix.go:238 +0x27 net.(TCPListener).accept(0x9110010, 0x9120e74, 0x9120e78, 0xc) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock_posix.go:139 +0x27 net.(TCPListener).Accept(0x9110010, 0x87a6db8, 0x91f92c0, 0x889fbe0, 0x416c3f28) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock.go:261 +0x57 google.golang.org/grpc.(Server).Serve(0x91f92c0, 0x8899780, 0x9110010, 0x0, 0x0) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.5.1/server.go:444 +0x14f github.com/osrg/gobgp/pkg/server.(server).serve.func1(0x8899780, 0x9110010) /home/runner/work/gobgp/gobgp/pkg/server/grpc_server.go:89 +0x78 created by github.com/osrg/gobgp/pkg/server.(*server).serve /home/runner/work/gobgp/gobgp/pkg/server/grpc_server.go:98 +0x440
goroutine 21 [IO wait, 100 minutes]: internal/poll.runtime_pollWait(0xe73d6d54, 0x72, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/netpoll.go:203 +0x4b internal/poll.(pollDesc).wait(0x90b2384, 0x72, 0x0, 0x0, 0x8773366) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:87 +0x37 internal/poll.(pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0x90b2370, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_unix.go:384 +0x19c net.(netFD).accept(0x90b2370, 0x85ec37d, 0x9322440, 0xc) /opt/hostedtoolcache/go/1.14.10/x64/src/net/fd_unix.go:238 +0x27 net.(TCPListener).accept(0x90a2ff0, 0x20, 0x90ba1ac, 0x90ba1a4) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock_posix.go:139 +0x27 net.(TCPListener).AcceptTCP(0x90a2ff0, 0x90ca758, 0x90ba180, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/tcpsock.go:248 +0x4f github.com/osrg/gobgp/pkg/server.newTCPListener.func1(0x90a2ff0, 0x9094880, 0x9322480, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/server.go:83 +0x54 created by github.com/osrg/gobgp/pkg/server.newTCPListener /home/runner/work/gobgp/gobgp/pkg/server/server.go:81 +0x1fd
goroutine 1763 [select]: github.com/eapache/channels.(*InfiniteChannel).infiniteBuffer(0x7c577a30) /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:49 +0xe4 created by github.com/eapache/channels.NewInfiniteChannel /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:19 +0x126
goroutine 23 [select]: github.com/eapache/channels.(*InfiniteChannel).infiniteBuffer(0x90a3230) /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:49 +0xe4 created by github.com/eapache/channels.NewInfiniteChannel /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:19 +0x126
goroutine 148 [select, 100 minutes]: github.com/eapache/channels.(*InfiniteChannel).infiniteBuffer(0x9184a40) /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:49 +0xe4 created by github.com/eapache/channels.NewInfiniteChannel /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:19 +0x126
goroutine 26 [runnable]: github.com/eapache/channels.(*InfiniteChannel).infiniteBuffer(0x90a3420) /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:49 +0xe4 created by github.com/eapache/channels.NewInfiniteChannel /home/runner/go/pkg/mod/github.com/eapache/channels@v1.1.0/infinite_channel.go:19 +0x126
goroutine 149 [select]: github.com/osrg/gobgp/pkg/server.(fsmHandler).established(0x9258300, 0x889a560, 0x925a020, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1757 +0x32b github.com/osrg/gobgp/pkg/server.(fsmHandler).loop(0x9258300, 0x889a560, 0x925a020, 0x9190ea0, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1870 +0x851 created by github.com/osrg/gobgp/pkg/server.newFSMHandler /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:406 +0x163
goroutine 151 [IO wait]: internal/poll.runtime_pollWait(0xe73d6c2c, 0x72, 0xffffffff) /opt/hostedtoolcache/go/1.14.10/x64/src/runtime/netpoll.go:203 +0x4b internal/poll.(pollDesc).wait(0x91ac834, 0x72, 0x0, 0x13, 0xffffffff) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:87 +0x37 internal/poll.(pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0x91ac820, 0x6bb36060, 0x13, 0x13, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/internal/poll/fd_unix.go:169 +0x168 net.(netFD).Read(0x91ac820, 0x6bb36060, 0x13, 0x13, 0x8890ea0, 0x7ff, 0x1) /opt/hostedtoolcache/go/1.14.10/x64/src/net/fd_unix.go:202 +0x3f net.(conn).Read(0x9186e10, 0x6bb36060, 0x13, 0x13, 0x0, 0x0, 0x0) /opt/hostedtoolcache/go/1.14.10/x64/src/net/net.go:184 +0x67 io.ReadAtLeast(0x8890ea0, 0x9186e10, 0x6bb36060, 0x13, 0x13, 0x13, 0x86440b6f, 0x90d2500, 0x85ac0c9) /opt/hostedtoolcache/go/1.14.10/x64/src/io/io.go:310 +0x74 io.ReadFull(...) /opt/hostedtoolcache/go/1.14.10/x64/src/io/io.go:329 github.com/osrg/gobgp/pkg/server.readAll(0x889fbe0, 0x9186e10, 0x13, 0x0, 0x66883868, 0x1, 0x1, 0x975481e0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:796 +0x85 github.com/osrg/gobgp/pkg/server.(fsmHandler).recvMessageWithError(0x9258300, 0x0, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:933 +0x6e github.com/osrg/gobgp/pkg/server.(fsmHandler).recvMessageloop(0x9258300, 0x889a560, 0x925a020, 0x9190f20, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1721 +0x5e created by github.com/osrg/gobgp/pkg/server.(fsmHandler).established /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1743 +0x175
goroutine 150 [select]: github.com/osrg/gobgp/pkg/server.(fsmHandler).sendMessageloop(0x9258300, 0x889a560, 0x925a020, 0x9190f20, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1683 +0x201 created by github.com/osrg/gobgp/pkg/server.(fsmHandler).established /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1741 +0x122
goroutine 1753 [select]: github.com/osrg/gobgp/pkg/server.(fsmHandler).connectLoop(0x1947c740, 0x889a560, 0x9aea23e0, 0x9aa17b90) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:526 +0x335 created by github.com/osrg/gobgp/pkg/server.(fsmHandler).active /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:602 +0x1080
goroutine 1764 [select]: github.com/osrg/gobgp/pkg/server.(fsmHandler).active(0x1947c740, 0x889a560, 0x9acbec40, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:611 +0x25f github.com/osrg/gobgp/pkg/server.(fsmHandler).loop(0x1947c740, 0x889a560, 0x9acbec40, 0x9a8f2940, 0x0, 0x0) /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:1864 +0x7da created by github.com/osrg/gobgp/pkg/server.newFSMHandler /home/runner/work/gobgp/gobgp/pkg/server/fsm.go:406 +0x163