search

Ingensi / dockbeat

Dockbeat - the elastic Beat for docker daemon monitoring
Other
286 stars 70 forks source link

Fatal error: concurrent map writes #90

Closed solomkinmv closed 8 years ago

solomkinmv commented 8 years ago

I've just launched your last development version and in a few minutes it threw following exception:

$ sudo ./dockerbeat
fatal error: concurrent map writes

goroutine 2529 [running]:
runtime.throw(0xa32d20, 0x15)
    /usr/local/go/src/runtime/panic.go:530 +0x90 fp=0xc8203c3b00 sp=0xc8203c3ae8
runtime.mapassign1(0x8459a0, 0xc82018e720, 0xc82008d290, 0xc8203c3d90)
    /usr/local/go/src/runtime/hashmap.go:445 +0xb1 fp=0xc8203c3ba8 sp=0xc8203c3b00
github.com/ingensi/dockerbeat/beat.(*EventGenerator).getBlkioEvent(0xc82008c0d8, 0xc82008d290, 0xc820211b80, 0xc82032ff20)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/EventGenerator.go:225 +0x93f fp=0xc8203c3e28 sp=0xc8203c3ba8
github.com/ingensi/dockerbeat/beat.(*Dockerbeat).exportContainerStats.func2(0xc82034efc0, 0xc82034f080, 0xc82008c090, 0xc82008d290)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/Dockerbeat.go:184 +0x1c5 fp=0xc8203c3f90 sp=0xc8203c3e28
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc8203c3f98 sp=0xc8203c3f90
created by github.com/ingensi/dockerbeat/beat.(*Dockerbeat).exportContainerStats
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/Dockerbeat.go:197 +0x210

goroutine 1 [chan receive, 22 minutes]:
github.com/elastic/beats/libbeat/beat.Run(0x9e07e0, 0xa, 0x9d6d30, 0xb, 0x7fe11aaa0bd0, 0xc82008c090, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:126 +0xc2
main.main()
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/main.go:14 +0xa5

goroutine 17 [syscall, 22 minutes, locked to thread]:
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1

goroutine 5 [syscall, 22 minutes]:
os/signal.signal_recv(0x0)
    /usr/local/go/src/runtime/sigqueue.go:116 +0x132
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
    /usr/local/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 34 [select]:
github.com/elastic/beats/libbeat/publisher.(*messageWorker).run(0xc82016e300)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/worker.go:61 +0x44a
created by github.com/elastic/beats/libbeat/publisher.(*messageWorker).init
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/worker.go:55 +0x11e

goroutine 7 [select]:
github.com/ingensi/dockerbeat/beat.(*Dockerbeat).Run(0xc82008c090, 0xc82008c120, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/Dockerbeat.go:101 +0x4e8
github.com/elastic/beats/libbeat/beat.(*Beat).Run(0xc82008c120, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:250 +0x335
github.com/elastic/beats/libbeat/beat.(*Beat).Start(0xc82008c120, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:162 +0x10f
github.com/elastic/beats/libbeat/beat.Run.func1(0xc82008c120)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:110 +0x42
created by github.com/elastic/beats/libbeat/beat.Run
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:122 +0x8d

goroutine 35 [select]:
github.com/elastic/beats/libbeat/publisher.(*bulkWorker).run(0xc8201d2000)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/bulk.go:57 +0x3cb
created by github.com/elastic/beats/libbeat/publisher.newBulkWorker
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/bulk.go:41 +0x20a

goroutine 36 [select, 22 minutes, locked to thread]:
runtime.gopark(0xacf2b8, 0xc8201d8f28, 0x9d2b78, 0x6, 0x18, 0x2)
    /usr/local/go/src/runtime/proc.go:262 +0x163
runtime.selectgoImpl(0xc8201d8f28, 0x0, 0x18)
    /usr/local/go/src/runtime/select.go:392 +0xa67
runtime.selectgo(0xc8201d8f28)
    /usr/local/go/src/runtime/select.go:215 +0x12
runtime.ensureSigM.func1()
    /usr/local/go/src/runtime/signal1_unix.go:279 +0x358
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1

goroutine 37 [chan receive, 22 minutes]:
github.com/elastic/beats/libbeat/service.HandleSignals.func1(0xc8201a30e0, 0xc82018cb40, 0xc82018cb30)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/service/service.go:29 +0x3a
created by github.com/elastic/beats/libbeat/service.HandleSignals
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/service/service.go:32 +0x21e

goroutine 133 [select]:
net/http.(*persistConn).writeLoop(0xc8202ac1a0)
    /usr/local/go/src/net/http/transport.go:1273 +0x472
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:854 +0x10cb

goroutine 132 [IO wait]:
net.runtime_pollWait(0x7fe11aaa1478, 0x72, 0xc820229000)
    /usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc820054300, 0x72, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc820054300, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8200542a0, 0xc820229000, 0x1000, 0x1000, 0x0, 0x7fe11aa9c028, 0xc82000a120)
    /usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc820028020, 0xc820229000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:172 +0xe4
net/http.noteEOFReader.Read(0x7fe11aa202c0, 0xc820028020, 0xc8202ac208, 0xc820229000, 0x1000, 0x1000, 0x404df3, 0x0, 0x0)
    /usr/local/go/src/net/http/transport.go:1683 +0x67
net/http.(*noteEOFReader).Read(0xc820168ac0, 0xc820229000, 0x1000, 0x1000, 0xc8203c7d1d, 0x0, 0x0)
    <autogenerated>:284 +0xd0
bufio.(*Reader).fill(0xc820184420)
    /usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc820184420, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:132 +0xcc
net/http.(*persistConn).readLoop(0xc8202ac1a0)
    /usr/local/go/src/net/http/transport.go:1069 +0x177
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:853 +0x10a6

goroutine 1875 [select]:
net/http.(*persistConn).writeLoop(0xc820246270)
    /usr/local/go/src/net/http/transport.go:1273 +0x472
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:854 +0x10cb

goroutine 1874 [IO wait]:
net.runtime_pollWait(0x7fe11aaa1538, 0x72, 0xc820295000)
    /usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc820121fe0, 0x72, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc820121fe0, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc820121f80, 0xc820295000, 0x1000, 0x1000, 0x0, 0x7fe11aa9c028, 0xc82000a120)
    /usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc82025eb40, 0xc820295000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:172 +0xe4
net/http.noteEOFReader.Read(0x7fe11aa5c000, 0xc82025eb40, 0xc8202462d8, 0xc820295000, 0x1000, 0x1000, 0x404df3, 0x0, 0x0)
    /usr/local/go/src/net/http/transport.go:1683 +0x67
net/http.(*noteEOFReader).Read(0xc820446960, 0xc820295000, 0x1000, 0x1000, 0xc820335d1d, 0x0, 0x0)
    <autogenerated>:284 +0xd0
bufio.(*Reader).fill(0xc820263d40)
    /usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc820263d40, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:132 +0xcc
net/http.(*persistConn).readLoop(0xc820246270)
    /usr/local/go/src/net/http/transport.go:1069 +0x177
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:853 +0x10a6
[cloud-user@mi-worker-001 dockerbeat]$ sudo ./dockerbeat
fatal error: concurrent map writes

goroutine 924 [running]:
runtime.throw(0xa32d20, 0x15)
    /usr/local/go/src/runtime/panic.go:530 +0x90 fp=0xc820247b00 sp=0xc820247ae8
runtime.mapassign1(0x8459a0, 0xc820061f50, 0xc8202cefc0, 0xc820247d90)
    /usr/local/go/src/runtime/hashmap.go:445 +0xb1 fp=0xc820247ba8 sp=0xc820247b00
github.com/ingensi/dockerbeat/beat.(*EventGenerator).getBlkioEvent(0xc820176048, 0xc8202cefc0, 0xc82030eb00, 0xc8202ca390)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/EventGenerator.go:225 +0x93f fp=0xc820247e28 sp=0xc820247ba8
github.com/ingensi/dockerbeat/beat.(*Dockerbeat).exportContainerStats.func2(0xc8202686c0, 0xc820268780, 0xc820176000, 0xc8202cefc0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/Dockerbeat.go:184 +0x1c5 fp=0xc820247f90 sp=0xc820247e28
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc820247f98 sp=0xc820247f90
created by github.com/ingensi/dockerbeat/beat.(*Dockerbeat).exportContainerStats
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/Dockerbeat.go:197 +0x210

goroutine 1 [chan receive, 7 minutes]:
github.com/elastic/beats/libbeat/beat.Run(0x9e07e0, 0xa, 0x9d6d30, 0xb, 0x7f7ac2b291e0, 0xc820176000, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:126 +0xc2
main.main()
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/main.go:14 +0xa5

goroutine 17 [syscall, 7 minutes, locked to thread]:
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1

goroutine 5 [syscall, 7 minutes]:
os/signal.signal_recv(0x0)
    /usr/local/go/src/runtime/sigqueue.go:116 +0x132
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
    /usr/local/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 18 [select, 1 minutes]:
github.com/ingensi/dockerbeat/beat.(*Dockerbeat).Run(0xc820176000, 0xc820176090, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/Dockerbeat.go:101 +0x4e8
github.com/elastic/beats/libbeat/beat.(*Beat).Run(0xc820176090, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:250 +0x335
github.com/elastic/beats/libbeat/beat.(*Beat).Start(0xc820176090, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:162 +0x10f
github.com/elastic/beats/libbeat/beat.Run.func1(0xc820176090)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:110 +0x42
created by github.com/elastic/beats/libbeat/beat.Run
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:122 +0x8d

goroutine 22 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7f7ac2b29cc8, 0x72, 0xc8201c4000)
    /usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc820147f70, 0x72, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc820147f70, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc820147f10, 0xc8201c4000, 0x1000, 0x1000, 0x0, 0x7f7ac2ba9028, 0xc82000a120)
    /usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc82013c0e8, 0xc8201c4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:172 +0xe4
net/http.noteEOFReader.Read(0x7f7ac2b2a608, 0xc82013c0e8, 0xc820145cd8, 0xc8201c4000, 0x1000, 0x1000, 0x404df3, 0x0, 0x0)
    /usr/local/go/src/net/http/transport.go:1683 +0x67
net/http.(*noteEOFReader).Read(0xc8201c20a0, 0xc8201c4000, 0x1000, 0x1000, 0xc820034d1d, 0x0, 0x0)
    <autogenerated>:284 +0xd0
bufio.(*Reader).fill(0xc820143740)
    /usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc820143740, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:132 +0xcc
net/http.(*persistConn).readLoop(0xc820145c70)
    /usr/local/go/src/net/http/transport.go:1069 +0x177
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:853 +0x10a6

goroutine 23 [select, 1 minutes]:
net/http.(*persistConn).writeLoop(0xc820145c70)
    /usr/local/go/src/net/http/transport.go:1273 +0x472
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:854 +0x10cb

goroutine 7 [select, 1 minutes]:
github.com/elastic/beats/libbeat/publisher.(*messageWorker).run(0xc820001b00)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/worker.go:61 +0x44a
created by github.com/elastic/beats/libbeat/publisher.(*messageWorker).init
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/worker.go:55 +0x11e

goroutine 8 [runnable]:
github.com/elastic/beats/libbeat/publisher.(*bulkWorker).onEvents(0xc820054d20, 0x0, 0x0, 0xc8202be4a0, 0x4, 0x4)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/bulk.go:83
github.com/elastic/beats/libbeat/publisher.(*bulkWorker).run(0xc820054d20)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/bulk.go:63 +0x352
created by github.com/elastic/beats/libbeat/publisher.newBulkWorker
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/bulk.go:41 +0x20a

goroutine 9 [select, 7 minutes, locked to thread]:
runtime.gopark(0xacf2b8, 0xc820027f28, 0x9d2b78, 0x6, 0x18, 0x2)
    /usr/local/go/src/runtime/proc.go:262 +0x163
runtime.selectgoImpl(0xc820027f28, 0x0, 0x18)
    /usr/local/go/src/runtime/select.go:392 +0xa67
runtime.selectgo(0xc820027f28)
    /usr/local/go/src/runtime/select.go:215 +0x12
runtime.ensureSigM.func1()
    /usr/local/go/src/runtime/signal1_unix.go:279 +0x358
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1

goroutine 10 [chan receive, 7 minutes]:
github.com/elastic/beats/libbeat/service.HandleSignals.func1(0xc82005f3e0, 0xc82021a250, 0xc82021a240)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/service/service.go:29 +0x3a
created by github.com/elastic/beats/libbeat/service.HandleSignals
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/service/service.go:32 +0x21e

goroutine 79 [select, 1 minutes]:
net/http.(*persistConn).writeLoop(0xc8202540d0)
    /usr/local/go/src/net/http/transport.go:1273 +0x472
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:854 +0x10cb

goroutine 78 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7f7ac2b29c08, 0x72, 0xc8204ba000)
    /usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc8204a2300, 0x72, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8204a2300, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8204a22a0, 0xc8204ba000, 0x1000, 0x1000, 0x0, 0x7f7ac2ba9028, 0xc82000a120)
    /usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc820028658, 0xc8204ba000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:172 +0xe4
net/http.noteEOFReader.Read(0x7f7ac2bb1c20, 0xc820028658, 0xc820254138, 0xc8204ba000, 0x1000, 0x1000, 0x404df3, 0x0, 0x0)
    /usr/local/go/src/net/http/transport.go:1683 +0x67
net/http.(*noteEOFReader).Read(0xc8200c08c0, 0xc8204ba000, 0x1000, 0x1000, 0xc820256d1d, 0x0, 0x0)
    <autogenerated>:284 +0xd0
bufio.(*Reader).fill(0xc82043f860)
    /usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc82043f860, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:132 +0xcc
net/http.(*persistConn).readLoop(0xc8202540d0)
    /usr/local/go/src/net/http/transport.go:1069 +0x177
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:853 +0x10a6
[cloud-user@mi-worker-001 dockerbeat]$ sudo ./dockerbeat
fatal error: concurrent map writes

goroutine 1323 [running]:
runtime.throw(0xa32d20, 0x15)
    /usr/local/go/src/runtime/panic.go:530 +0x90 fp=0xc820269b00 sp=0xc820269ae8
runtime.mapassign1(0x8459a0, 0xc8201c0720, 0xc82017e750, 0xc820269d90)
    /usr/local/go/src/runtime/hashmap.go:445 +0xb1 fp=0xc820269ba8 sp=0xc820269b00
github.com/ingensi/dockerbeat/beat.(*EventGenerator).getBlkioEvent(0xc82016c048, 0xc82017e750, 0xc8201a18c0, 0xc82012e9f0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/EventGenerator.go:225 +0x93f fp=0xc820269e28 sp=0xc820269ba8
github.com/ingensi/dockerbeat/beat.(*Dockerbeat).exportContainerStats.func2(0xc820523f80, 0xc8200da060, 0xc82016c000, 0xc82017e750)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/Dockerbeat.go:184 +0x1c5 fp=0xc820269f90 sp=0xc820269e28
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc820269f98 sp=0xc820269f90
created by github.com/ingensi/dockerbeat/beat.(*Dockerbeat).exportContainerStats
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/Dockerbeat.go:197 +0x210

goroutine 1 [chan receive, 11 minutes]:
github.com/elastic/beats/libbeat/beat.Run(0x9e07e0, 0xa, 0x9d6d30, 0xb, 0x7fe48f96f150, 0xc82016c000, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:126 +0xc2
main.main()
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/main.go:14 +0xa5

goroutine 17 [syscall, 11 minutes, locked to thread]:
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1

goroutine 5 [syscall, 11 minutes]:
os/signal.signal_recv(0x0)
    /usr/local/go/src/runtime/sigqueue.go:116 +0x132
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
    /usr/local/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 34 [select]:
github.com/elastic/beats/libbeat/publisher.(*messageWorker).run(0xc820060300)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/worker.go:61 +0x44a
created by github.com/elastic/beats/libbeat/publisher.(*messageWorker).init
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/worker.go:55 +0x11e

goroutine 18 [select]:
github.com/ingensi/dockerbeat/beat.(*Dockerbeat).Run(0xc82016c000, 0xc82016c090, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/beat/Dockerbeat.go:101 +0x4e8
github.com/elastic/beats/libbeat/beat.(*Beat).Run(0xc82016c090, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:250 +0x335
github.com/elastic/beats/libbeat/beat.(*Beat).Start(0xc82016c090, 0x0, 0x0)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:162 +0x10f
github.com/elastic/beats/libbeat/beat.Run.func1(0xc82016c090)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:110 +0x42
created by github.com/elastic/beats/libbeat/beat.Run
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/beat/beat.go:122 +0x8d

goroutine 35 [runnable]:
github.com/elastic/beats/libbeat/publisher.(*bulkWorker).run(0xc820204000)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/bulk.go:57 +0x134
created by github.com/elastic/beats/libbeat/publisher.newBulkWorker
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/publisher/bulk.go:41 +0x20a

goroutine 36 [select, 11 minutes, locked to thread]:
runtime.gopark(0xacf2b8, 0xc82020af28, 0x9d2b78, 0x6, 0x18, 0x2)
    /usr/local/go/src/runtime/proc.go:262 +0x163
runtime.selectgoImpl(0xc82020af28, 0x0, 0x18)
    /usr/local/go/src/runtime/select.go:392 +0xa67
runtime.selectgo(0xc82020af28)
    /usr/local/go/src/runtime/select.go:215 +0x12
runtime.ensureSigM.func1()
    /usr/local/go/src/runtime/signal1_unix.go:279 +0x358
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1

goroutine 37 [chan receive, 11 minutes]:
github.com/elastic/beats/libbeat/service.HandleSignals.func1(0xc8201d50e0, 0xc8201beb40, 0xc8201beb30)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/service/service.go:29 +0x3a
created by github.com/elastic/beats/libbeat/service.HandleSignals
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/elastic/beats/libbeat/service/service.go:32 +0x21e

goroutine 123 [select]:
net/http.(*persistConn).writeLoop(0xc8204d40d0)
    /usr/local/go/src/net/http/transport.go:1273 +0x472
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:854 +0x10cb

goroutine 122 [IO wait]:
net.runtime_pollWait(0x7fe48f96fb78, 0x72, 0xc82015a000)
    /usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc820054290, 0x72, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc820054290, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc820054230, 0xc82015a000, 0x1000, 0x1000, 0x0, 0x7fe48f9af028, 0xc82000a120)
    /usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc8202de000, 0xc82015a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:172 +0xe4
net/http.noteEOFReader.Read(0x7fe48f9706c8, 0xc8202de000, 0xc8204d4138, 0xc82015a000, 0x1000, 0x1000, 0x404df3, 0x0, 0x0)
    /usr/local/go/src/net/http/transport.go:1683 +0x67
net/http.(*noteEOFReader).Read(0xc8201b8280, 0xc82015a000, 0x1000, 0x1000, 0xc82026ad1d, 0x0, 0x0)
    <autogenerated>:284 +0xd0
bufio.(*Reader).fill(0xc8201d44e0)
    /usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc8201d44e0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:132 +0xcc
net/http.(*persistConn).readLoop(0xc8204d40d0)
    /usr/local/go/src/net/http/transport.go:1069 +0x177
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:853 +0x10a6

goroutine 941 [select]:
net/http.(*persistConn).writeLoop(0xc8204d41a0)
    /usr/local/go/src/net/http/transport.go:1273 +0x472
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:854 +0x10cb

goroutine 940 [IO wait]:
net.runtime_pollWait(0x7fe48f96fc38, 0x72, 0xc8202cd000)
    /usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc820205800, 0x72, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc820205800, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8202057a0, 0xc8202cd000, 0x1000, 0x1000, 0x0, 0x7fe48f9af028, 0xc82000a120)
    /usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc8202df930, 0xc8202cd000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:172 +0xe4
net/http.noteEOFReader.Read(0x7fe48f970578, 0xc8202df930, 0xc8204d4208, 0xc8202cd000, 0x1000, 0x1000, 0x404df3, 0x0, 0x0)
    /usr/local/go/src/net/http/transport.go:1683 +0x67
net/http.(*noteEOFReader).Read(0xc8202d1020, 0xc8202cd000, 0x1000, 0x1000, 0xc8203cfd1d, 0x0, 0x0)
    <autogenerated>:284 +0xd0
bufio.(*Reader).fill(0xc82040be00)
    /usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc82040be00, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:132 +0xcc
net/http.(*persistConn).readLoop(0xc8204d41a0)
    /usr/local/go/src/net/http/transport.go:1069 +0x177
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:853 +0x10a6

goroutine 1339 [runnable]:
github.com/fsouza/go-dockerclient.(*Client).Stats.func3(0xc8201ad300, 0x40, 0xc8200da0c0, 0xc8200da000, 0xc8200da120, 0xffffffffffffffff, 0xc8202de670, 0xc82017cc00)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/container.go:798 +0xe4
created by github.com/fsouza/go-dockerclient.(*Client).Stats
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/container.go:804 +0x3a7

goroutine 1327 [runnable]:
github.com/fsouza/go-dockerclient.(*Client).Stats.func3(0xc8201ad240, 0x40, 0xc820523f80, 0xc820523f00, 0xc8200da000, 0xffffffffffffffff, 0xc820134cd8, 0xc8200da240)
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/container.go:798 +0xe4
created by github.com/fsouza/go-dockerclient.(*Client).Stats
    /home/cloud-user/.go/src/github.com/ingensi/dockerbeat/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/container.go:804 +0x3a7
marminthibaut commented 8 years ago

Hi,

can you be more precise? Context, env, etc.

Did it work fine before the update?

Thanks

solomkinmv commented 8 years ago

CentOS 7, Docker 1.10.3, Go 1.6. I've built binary from development branch. And it is my first usage of dockerbeat :)

marminthibaut commented 8 years ago

Ok thank I'll have a look.

You can try the beta2 version, waiting for the bug resolution.

solomkinmv commented 8 years ago

I've updated libbeat for beta 2 and got the same error. Any ideas?

marminthibaut commented 8 years ago

No idea.. We will work on it as soon as possible.

Thanks for the feedback

erwanncloarec commented 8 years ago

We have a concurrent map read/write problem here because we use a goroutine on exportContainerStats function.

Golang has made the choice to not define map operations as atomic. Therefore they provide a solution, using sync.RWMutex.

I think we should try that instead of a new structure for concurrent map.

solomkinmv commented 8 years ago

@erwanncloarec I'm a total newbie in Go, so it was first working solution for this problem :)

marminthibaut commented 8 years ago

I think problem comes from this line: https://github.com/Ingensi/dockerbeat/blob/develop/beat/EventGenerator.go#L227

I'll try a mutex based solution soon...