Open traetox opened 3 years ago
docker-ovs-plugin version 0.2
Looks like there are a few global maps that are not protected with mutex's
https://github.com/gopher-net/docker-ovs-plugin/blob/f58463cbc88140b256a78543cdefd7adaffd598f/ovs/ovsdb.go#L24
I was running an experiment with many containers starting and stopping, the plugin panic'd with the following backtrace:
fatal error: concurrent map writes goroutine 3521 [running]: runtime.throw(0x8283a9, 0x15) /usr/local/go/src/runtime/panic.go:566 +0x95 fp=0xc4202136d8 sp=0xc4202136b8 runtime.mapassign1(0x7b2e40, 0xc4200dcea0, 0xc420213830, 0xc420213860) /usr/local/go/src/runtime/hashmap.go:458 +0x8ef fp=0xc4202137c0 sp=0xc4202136d8 github.com/gopher-net/docker-ovs-plugin/ovs.populateCache(0xc420110c60) /go/src/github.com/gopher-net/docker-ovs-plugin/ovs/ovsdb.go:156 +0x307 fp=0xc420213940 sp=0xc4202137c0 github.com/gopher-net/docker-ovs-plugin/ovs.OvsdbNotifier.Update(0x78c4a0, 0xc4204278c0, 0xc420110c60) /go/src/github.com/gopher-net/docker-ovs-plugin/ovs/ovsdb.go:36 +0x5b fp=0xc420213978 sp=0xc420213940 github.com/gopher-net/docker-ovs-plugin/ovs.(*OvsdbNotifier).Update(0xa0a810, 0x78c4a0, 0xc4204278c0, 0xc420110c60) <autogenerated>:1 +0x6a fp=0xc4202139b8 sp=0xc420213978 github.com/socketplane/libovsdb.update(0xc4200de0e0, 0xc420180300, 0x2, 0x4, 0xc420500230, 0x0, 0x0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/socketplane/libovsdb/client.go:133 +0x24b fp=0xc420213a68 sp=0xc4202139b8 runtime.call64(0xc4200dc330, 0x858e90, 0xc4201803c0, 0x2800000038) /usr/local/go/src/runtime/asm_amd64.s:480 +0x4c fp=0xc420213ab8 sp=0xc420213a68 reflect.Value.call(0x7af420, 0x858e90, 0x13, 0x821de5, 0x4, 0xc420213ee0, 0x3, 0x3, 0x0, 0x7f7940, ...) /usr/local/go/src/reflect/value.go:434 +0x5c8 fp=0xc420213e08 sp=0xc420213ab8 reflect.Value.Call(0x7af420, 0x858e90, 0x13, 0xc420213ee0, 0x3, 0x3, 0x0, 0x0, 0x0) /usr/local/go/src/reflect/value.go:302 +0xa4 fp=0xc420213e70 sp=0xc420213e08 github.com/cenkalti/rpc2.(*Client).readRequest.func1(0xc4200e2080, 0xc4200de0e0, 0x78c4a0, 0xc42019e2e0, 0x197, 0x7806c0, 0xc420500230, 0x16, 0x0, 0xc42022d7c8, ...) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/cenkalti/rpc2/client.go:151 +0x133 fp=0xc420213f38 sp=0xc420213e70 runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc420213f40 sp=0xc420213f38 created by github.com/cenkalti/rpc2.(*Client).readRequest /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/cenkalti/rpc2/client.go:169 +0x25b goroutine 1 [IO wait, 87 minutes]: net.runtime_pollWait(0x7ff12bfc4f18, 0x72, 0x0) /usr/local/go/src/runtime/netpoll.go:160 +0x59 net.(*pollDesc).wait(0xc420058bc0, 0x72, 0xc42010b8f8, 0xc420012160) /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 net.(*pollDesc).waitRead(0xc420058bc0, 0x9cd380, 0xc420012160) /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 net.(*netFD).accept(0xc420058b60, 0x0, 0x9cbc40, 0xc4201433e0) /usr/local/go/src/net/fd_unix.go:419 +0x238 net.(*UnixListener).accept(0xc420143160, 0xc420132cf0, 0xc42010b9e0, 0x66f69f) /usr/local/go/src/net/unixsock_posix.go:158 +0x32 net.(*UnixListener).Accept(0xc420143160, 0xc42010ba30, 0xc42010ba38, 0xc42010ba28, 0x5922dd) /usr/local/go/src/net/unixsock.go:229 +0x49 github.com/docker/docker/pkg/listenbuffer.(*defaultListener).Accept(0xc420143180, 0x8590b8, 0xc42000f180, 0x9d0680, 0xc420132cf0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/docker/docker/pkg/listenbuffer/buffer.go:71 +0x3f net/http.(*Server).Serve(0xc42000ef00, 0x9cfc40, 0xc420143180, 0x0, 0x0) /usr/local/go/src/net/http/server.go:2273 +0x1ce github.com/gopher-net/dknet.(*Handler).listenAndServe(0xc420143060, 0x822069, 0x4, 0x821cb5, 0x3, 0x822025, 0x4, 0x0, 0x0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/gopher-net/dknet/api.go:283 +0x1dc github.com/gopher-net/dknet.(*Handler).ServeUnix(0xc420143060, 0x822025, 0x4, 0x821cb5, 0x3, 0x7, 0xc4200a0700) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/gopher-net/dknet/api.go:252 +0x68 main.Run(0xc4200a0790) /go/src/github.com/gopher-net/docker-ovs-plugin/main.go:44 +0xb6 github.com/codegangsta/cli.(*App).Run(0xc4200ca0c0, 0xc42000c0e0, 0x2, 0x2, 0xc420063950, 0x4052eb) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/codegangsta/cli/app.go:132 +0x5f8 main.main() /go/src/github.com/gopher-net/docker-ovs-plugin/main.go:30 +0x19b goroutine 17 [syscall, 87 minutes, locked to thread]: runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1 goroutine 18 [IO wait]: net.runtime_pollWait(0x7ff12bfc4fd8, 0x72, 0x3) /usr/local/go/src/runtime/netpoll.go:160 +0x59 net.(*pollDesc).wait(0xc4200de0d0, 0x72, 0xc42018fba8, 0xc420012160) /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 net.(*pollDesc).waitRead(0xc4200de0d0, 0x9cd380, 0xc420012160) /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 net.(*netFD).Read(0xc4200de070, 0xc42050a000, 0xfe00, 0xfe00, 0x0, 0x9cd380, 0xc420012160) /usr/local/go/src/net/fd_unix.go:243 +0x1a1 net.(*conn).Read(0xc4200e0000, 0xc42050a000, 0xfe00, 0xfe00, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:173 +0x70 encoding/json.(*Decoder).refill(0xc4200e4000, 0x7a98a0, 0xc42019e201) /usr/local/go/src/encoding/json/stream.go:152 +0xfa encoding/json.(*Decoder).readValue(0xc4200e4000, 0x0, 0x0, 0x671ccb) /usr/local/go/src/encoding/json/stream.go:128 +0x198 encoding/json.(*Decoder).Decode(0xc4200e4000, 0x77ea40, 0xc4200e8020, 0xc420500230, 0x16) /usr/local/go/src/encoding/json/stream.go:57 +0x8e github.com/cenkalti/rpc2/jsonrpc.(*jsonCodec).ReadHeader(0xc4200e8000, 0xc4200c7740, 0xc4200c7760, 0x0, 0x0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/cenkalti/rpc2/jsonrpc/jsonrpc.go:93 +0x70 github.com/cenkalti/rpc2.(*Client).readLoop(0xc4200de0e0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/cenkalti/rpc2/client.go:79 +0x116 github.com/cenkalti/rpc2.(*Client).Run(0xc4200de0e0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/cenkalti/rpc2/client.go:54 +0x2b created by github.com/socketplane/libovsdb.Connect /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/socketplane/libovsdb/client.go:55 +0x631 goroutine 19 [chan receive, 87 minutes]: github.com/socketplane/libovsdb.handleDisconnectNotification(0xc4200de0e0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/socketplane/libovsdb/client.go:249 +0x49 created by github.com/socketplane/libovsdb.Connect /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/socketplane/libovsdb/client.go:56 +0x656 goroutine 6 [chan receive]: github.com/gopher-net/docker-ovs-plugin/ovs.(*ovsdber).monitorBridges(0xc420132318) /go/src/github.com/gopher-net/docker-ovs-plugin/ovs/ovsdb.go:122 +0x71 created by github.com/gopher-net/docker-ovs-plugin/ovs.(*ovsdber).initDBCache /go/src/github.com/gopher-net/docker-ovs-plugin/ovs/ovsdb.go:66 +0x2b9 goroutine 7 [syscall]: syscall.Syscall6(0x2c, 0x7, 0xc42052ee60, 0x20, 0x0, 0xc42052ee34, 0xc, 0xc420186e10, 0x10, 0xc42052ee60) /usr/local/go/src/syscall/asm_linux_amd64.s:44 +0x5 syscall.sendto(0x7, 0xc42052ee60, 0x20, 0x20, 0x0, 0xc42052ee34, 0xc40000000c, 0xc42052ee60, 0xc420384690) /usr/local/go/src/syscall/zsyscall_linux_amd64.go:1729 +0x80 syscall.Sendto(0x7, 0xc42052ee60, 0x20, 0x20, 0x0, 0x9cbf40, 0xc42052ee28, 0xc4204cd8f0, 0xc4204cd900) /usr/local/go/src/syscall/syscall_unix.go:265 +0x92 github.com/vishvananda/netlink/nl.(*NetlinkSocket).Send(0xc42052ee20, 0xc420186e10, 0xc42052ee20, 0x0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/vishvananda/netlink/nl/nl_linux.go:333 +0x7c github.com/vishvananda/netlink/nl.(*NetlinkRequest).Execute(0xc420186e10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/vishvananda/netlink/nl/nl_linux.go:215 +0xfd github.com/vishvananda/netlink.LinkDel(0x9ce200, 0xc42055ff80, 0x4, 0xc4200e27c0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/vishvananda/netlink/link_linux.go:432 +0xfe github.com/gopher-net/docker-ovs-plugin/ovs.(*Driver).Leave(0xc420132300, 0xc420186cf0, 0xc420186cf0, 0x0) /go/src/github.com/gopher-net/docker-ovs-plugin/ovs/driver.go:171 +0x1b6 github.com/gopher-net/dknet.(*Handler).initMux.func9(0x9cfec0, 0xc420065ee0, 0xc42034a0f0) /go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/gopher-net/dknet/api.go:233 +0xb4 net/http.HandlerFunc.ServeHTTP(0xc420136dc0, 0x9cfec0, 0xc420065ee0, 0xc42034a0f0) /usr/local/go/src/net/http/server.go:1726 +0x44 net/http.(*ServeMux).ServeHTTP(0xc420132c00, 0x9cfec0, 0xc420065ee0, 0xc42034a0f0) /usr/local/go/src/net/http/server.go:2022 +0x7f net/http.serverHandler.ServeHTTP(0xc42000ef00, 0x9cfec0, 0xc420065ee0, 0xc42034a0f0) /usr/local/go/src/net/http/server.go:2202 +0x7d net/http.(*conn).serve(0xc42000f180, 0x9d05c0, 0xc42012d480) /usr/local/go/src/net/http/server.go:1579 +0x4b7 created by net/http.(*Server).Serve /usr/local/go/src/net/http/server.go:2293 +0x44d
docker-ovs-plugin version 0.2
Looks like there are a few global maps that are not protected with mutex's
https://github.com/gopher-net/docker-ovs-plugin/blob/f58463cbc88140b256a78543cdefd7adaffd598f/ovs/ovsdb.go#L24
I was running an experiment with many containers starting and stopping, the plugin panic'd with the following backtrace: