search

hyperhq / runv

Hypervisor-based Runtime for OCI
Apache License 2.0
828 stars 129 forks source link

left go routine after disassociation #508

Open laijs opened 7 years ago

laijs commented 7 years ago

it is better to cleanup them and make it clean state after disassociation

INFO[0083] === BEGIN goroutine stack dump ===
goroutine 28 [running]:
github.com/hyperhq/hyperd/vendor/github.com/docker/docker/pkg/signal.DumpStacks()
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/pkg/signal/trap.go:67 +0x8f
github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon.setupDumpStackTrap.func1(0xc4200ae0c0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/debugtrap_unix.go:18 +0x73
created by github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon.setupDumpStackTrap
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/debugtrap_unix.go:20 +0xec

goroutine 1 [chan receive]:
main.mainDaemon(0xc4210f9ec0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/hyperd.go:212 +0xb2e
main.main()
    /root/work/gopath/src/github.com/hyperhq/hyperd/hyperd.go:75 +0x41f

goroutine 17 [syscall, 1 minutes, locked to thread]:
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 5 [syscall]:
os/signal.signal_recv(0x1f489a0)
    /usr/local/go/src/runtime/sigqueue.go:116 +0x157
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
    /usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 6 [chan receive]:
github.com/hyperhq/hyperd/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x1f72460)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/golang/glog/glog.go:882 +0x7a
created by github.com/hyperhq/hyperd/vendor/github.com/golang/glog.init.1
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/golang/glog/glog.go:410 +0x21d

goroutine 23 [select]:
github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc42000e480)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x5e1
created by github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb.openDB
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x592

goroutine 9 [select]:
github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc420212620)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x247
created by github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x1bc

goroutine 24 [select]:
github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc42000e480)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:101 +0x249
created by github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb.openDB
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x5b4

goroutine 25 [select]:
github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc42000e480)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x83c
created by github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb.openDB
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x73a

goroutine 26 [select, 1 minutes]:
github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc42000e480)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x263
created by github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb.openDB
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x75c

goroutine 27 [select, locked to thread]:
runtime.gopark(0x16429f8, 0x0, 0x1560ff5, 0x6, 0x18, 0x2)
    /usr/local/go/src/runtime/proc.go:259 +0x13a
runtime.selectgoImpl(0xc420029f30, 0x0, 0x18)
    /usr/local/go/src/runtime/select.go:423 +0x11d9
runtime.selectgo(0xc420029f30)
    /usr/local/go/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
    /usr/local/go/src/runtime/signal1_unix.go:304 +0x2f3
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 29 [select, 1 minutes]:
github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork.(*controller).watchLoop(0xc4200cc420)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork/store.go:409 +0x151
created by github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork.(*controller).startWatch
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork/store.go:423 +0xff

goroutine 31 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7fc42b51e868, 0x72, 0xc420627688)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42072ed80, 0x72, 0xc4206277c8, 0xc4200101b0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42072ed80, 0x1f438e0, 0xc4200101b0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).readMsg(0xc42072ed20, 0xc420682160, 0x10, 0x10, 0xc420ffc420, 0x1000, 0x1000, 0xffffffffffffffff, 0x0, 0x0, ...)
    /usr/local/go/src/net/fd_unix.go:297 +0x243
net.(*UnixConn).readMsg(0xc42002a430, 0xc420682160, 0x10, 0x10, 0xc420ffc420, 0x1000, 0x1000, 0xc4206eb9c7, 0xc42072f960, 0x0, ...)
    /usr/local/go/src/net/unixsock_posix.go:111 +0x91
net.(*UnixConn).ReadMsgUnix(0xc42002a430, 0xc420682160, 0x10, 0x10, 0xc420ffc420, 0x1000, 0x1000, 0x0, 0x0, 0xc420682060, ...)
    /usr/local/go/src/net/unixsock.go:127 +0xaa
github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus.(*oobReader).Read(0xc420ffc400, 0xc420682160, 0x10, 0x10, 0x1020, 0xc420ffc400, 0x0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus/transport_unix.go:21 +0x95
io.ReadAtLeast(0x1f3de20, 0xc420ffc400, 0xc420682160, 0x10, 0x10, 0x10, 0x142bd00, 0x12dd201, 0xc420ffc400)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x1f3de20, 0xc420ffc400, 0xc420682160, 0x10, 0x10, 0x51, 0x0, 0x0)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus.(*unixTransport).ReadMessage(0xc4206ea730, 0xc420b98600, 0xc420627d34, 0x1f42020)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus/transport_unix.go:85 +0x15b
github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus.(*Conn).inWorker(0xc42006ed00)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus/conn.go:295 +0x4b
created by github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus.(*Conn).Auth
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus/auth.go:118 +0x738

goroutine 32 [chan receive, 1 minutes]:
github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus.(*Conn).outWorker(0xc42006ed00)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus/conn.go:437 +0x5d
created by github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus.(*Conn).Auth
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/godbus/dbus/auth.go:119 +0x75d

goroutine 50 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7fc42b51e7a8, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42072fb10, 0x72, 0xc42074c620, 0xc4200101b0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42072fb10, 0x1f438e0, 0xc4200101b0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc42072fab0, 0x0, 0x1f405a0, 0xc4206a4f20)
    /usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*UnixListener).accept(0xc4201af5a0, 0xc42074c7a0, 0xe8d4f7, 0xc420b93340)
    /usr/local/go/src/net/unixsock_posix.go:158 +0x32
net.(*UnixListener).Accept(0xc4201af5a0, 0x203070100000101, 0x14, 0xc, 0x14)
    /usr/local/go/src/net/unixsock.go:229 +0x49
github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork.(*controller).acceptClientConnections(0xc4200cc420, 0xc42072fa40, 0x63, 0x1f4d160, 0xc4201af5a0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork/sandbox_externalkey_unix.go:130 +0x43
created by github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork.(*controller).startExternalKeyListener
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork/sandbox_externalkey_unix.go:124 +0x228

goroutine 10 [chan receive, 1 minutes]:
github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork/iptables.signalHandler()
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork/iptables/firewalld.go:92 +0x59
created by github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork/iptables.FirewalldInit
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/libnetwork/iptables/firewalld.go:48 +0xa7

goroutine 51 [chan receive]:
github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon.(*statsCollector).run(0xc42064c680)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/stats_collector_unix.go:99 +0xc5
created by github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon.(*Daemon).newStatsCollector
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/stats_collector_unix.go:38 +0x177

goroutine 52 [chan receive, 1 minutes]:
github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon.(*Daemon).execCommandGC(0xc42000e180)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/exec.go:237 +0x8d
created by github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon.NewDaemon
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/daemon.go:819 +0x2151

goroutine 16 [sleep]:
time.Sleep(0x2540be400)
    /usr/local/go/src/runtime/time.go:59 +0xe1
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*VmContext).watchHyperstart(0xc420b82b00, 0x0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/hypervisor.go:59 +0x2ca
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*VmContext).Launch
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/hypervisor.go:76 +0x399

goroutine 120 [select]:
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*Vm).WaitResponse.func1(0xc420b52be0, 0xc420b86c00, 0xc420b86b40, 0xc421131260, 0xc420b86ba0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm.go:126 +0x2af
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*Vm).WaitResponse
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm.go:141 +0x118

goroutine 104 [chan receive]:
github.com/hyperhq/hyperd/daemon/pod.(*XPod).waitVMStop(0xc420b842c0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/daemon/pod/decommission.go:525 +0xc1
created by github.com/hyperhq/hyperd/daemon/pod.(*XPod).createSandbox
    /root/work/gopath/src/github.com/hyperhq/hyperd/daemon/pod/provision.go:271 +0x29d

goroutine 114 [chan receive]:
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.watchVmConsole(0xc420b82b00)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_console.go:94 +0x765
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*VmContext).Launch
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/hypervisor.go:79 +0x2a8

goroutine 118 [syscall]:
syscall.Syscall(0x0, 0x10, 0xc421132002, 0x5fe, 0x0, 0x800000, 0xc420a6bc78)
    /usr/local/go/src/syscall/asm_linux_amd64.s:18 +0x5
syscall.read(0x10, 0xc421132002, 0x5fe, 0x5fe, 0x72, 0x0, 0x0)
    /usr/local/go/src/syscall/zsyscall_linux_amd64.go:783 +0x55
syscall.Read(0x10, 0xc421132002, 0x5fe, 0x5fe, 0x0, 0x0, 0x0)
    /usr/local/go/src/syscall/syscall_unix.go:161 +0x49
net.(*netFD).Read(0xc420837730, 0xc421132002, 0x5fe, 0x5fe, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/fd_unix.go:239 +0x10f
net.(*conn).Read(0xc4200b51d0, 0xc421132002, 0x5fe, 0x5fe, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
encoding/json.(*Decoder).refill(0xc42111c340, 0xa, 0x9)
    /usr/local/go/src/encoding/json/stream.go:152 +0xfa
encoding/json.(*Decoder).readValue(0xc42111c340, 0x0, 0x0, 0x10)
    /usr/local/go/src/encoding/json/stream.go:128 +0x198
encoding/json.(*Decoder).Decode(0xc42111c340, 0x1368fa0, 0xc420815ca0, 0x0, 0x0)
    /usr/local/go/src/encoding/json/stream.go:57 +0x8e
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu.qmpReceiver(0xc420b862a0, 0xc420b7eaf0, 0xc42111c340)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu/qmp_handler.go:140 +0x12e
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu.qmpHandler.func3
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu/qmp_handler.go:352 +0x3e5

goroutine 152 [semacquire]:
sync.runtime_notifyListWait(0xc421169890, 0x0)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421169880)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/hyperhq/hyperd/vendor/github.com/docker/docker/pkg/ioutils.(*BytesPipe).Read(0xc421105ea0, 0xc421250000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/pkg/ioutils/bytespipe.go:123 +0xb2
io.copyBuffer(0x1f3f820, 0xc4200b5628, 0x7fc42b45b830, 0xc421105ea0, 0xc421250000, 0x8000, 0x8000, 0x140bf20, 0x0, 0x7fc42b45b830)
    /usr/local/go/src/io/io.go:390 +0x147
io.Copy(0x1f3f820, 0xc4200b5628, 0x7fc42b45b830, 0xc421105ea0, 0xc421105ea0, 0x0, 0x0)
    /usr/local/go/src/io/io.go:360 +0x68
github.com/hyperhq/hyperd/daemon/pod.(*Container).AttachStreams.func2(0x1561115, 0x6, 0x1f3f820, 0xc4200b5628, 0x1f45160, 0xc421105ea0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/daemon/pod/container.go:1358 +0x207
created by github.com/hyperhq/hyperd/daemon/pod.(*Container).AttachStreams
    /root/work/gopath/src/github.com/hyperhq/hyperd/daemon/pod/container.go:1379 +0x409

goroutine 151 [semacquire]:
sync.runtime_notifyListWait(0xc421169810, 0x0)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421169800)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/hyperhq/hyperd/vendor/github.com/docker/docker/pkg/ioutils.(*BytesPipe).Read(0xc421105e50, 0xc421258000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/pkg/ioutils/bytespipe.go:123 +0xb2
io.copyBuffer(0x1f3f820, 0xc4200b5618, 0x7fc42b45b830, 0xc421105e50, 0xc421258000, 0x8000, 0x8000, 0x140bf20, 0x0, 0x7fc42b45b830)
    /usr/local/go/src/io/io.go:390 +0x147
io.Copy(0x1f3f820, 0xc4200b5618, 0x7fc42b45b830, 0xc421105e50, 0xc421105e50, 0x0, 0x0)
    /usr/local/go/src/io/io.go:360 +0x68
github.com/hyperhq/hyperd/daemon/pod.(*Container).AttachStreams.func2(0x156111b, 0x6, 0x1f3f820, 0xc4200b5618, 0x1f45160, 0xc421105e50)
    /root/work/gopath/src/github.com/hyperhq/hyperd/daemon/pod/container.go:1358 +0x207
created by github.com/hyperhq/hyperd/daemon/pod.(*Container).AttachStreams
    /root/work/gopath/src/github.com/hyperhq/hyperd/daemon/pod/container.go:1378 +0x3a7

goroutine 38 [chan receive]:
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.(*jsonBasedHyperstart).WaitProcess(0xc420b86480, 0xc421168140, 0x40, 0x155e18b, 0x4, 0xc4211aa1a8)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:782 +0x10d
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*VmContext).newContainer.func1(0xc420b82b00, 0xc421168140, 0x40)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_states.go:49 +0x89
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*VmContext).newContainer
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_states.go:62 +0x5da

goroutine 149 [semacquire]:
sync.runtime_Semacquire(0xc42120753c)
    /usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc421207530)
    /usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.streamCopy(0xc42118fc50, 0x1f45b20, 0xc42121c310, 0x1f470e0, 0xc4200b5630, 0x1f470e0, 0xc4200b5640)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_states.go:214 +0x133
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*VmContext).attachCmd
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_states.go:139 +0x3fe

goroutine 101 [IO wait]:
net.runtime_pollWait(0x7fc42b51e268, 0x72, 0x17)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4209a98e0, 0x72, 0xc4207d0c60, 0xc4200101b0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4209a98e0, 0x1f438e0, 0xc4200101b0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4209a9880, 0xc420a2e400, 0xc, 0x200, 0x0, 0x1f438e0, 0xc4200101b0)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4201ac900, 0xc420a2e400, 0xc, 0x200, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.readTtyMessage(0x7fc419e66b68, 0xc4201ac900, 0x0, 0x0, 0x0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:368 +0x10e
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.handleStreamFromHyperstart(0xc420b86480, 0x7fc419e66b68, 0xc4201ac900)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:425 +0x7e
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.handleStreamSock
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:418 +0x344

goroutine 100 [chan receive]:
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.handleStreamToHyperstart(0xc420b86480, 0x7fc42b45b290, 0xc4201ac900)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:393 +0x63
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.handleStreamSock
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:417 +0x2e7

goroutine 99 [chan send]:
github.com/hyperhq/hyperd/server.(*Server).Wait(0xc4209c3e00, 0xc4209c3f20)
    /root/work/gopath/src/github.com/hyperhq/hyperd/server/server.go:222 +0x133
created by main.mainDaemon
    /root/work/gopath/src/github.com/hyperhq/hyperd/hyperd.go:182 +0x609

goroutine 35 [IO wait]:
net.runtime_pollWait(0x7fc42b51e328, 0x72, 0x16)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420819480, 0x72, 0xc42074f530, 0xc4200101b0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420819480, 0x1f438e0, 0xc4200101b0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420819420, 0xc420824300, 0x100, 0x100, 0x0, 0x1f438e0, 0xc4200101b0)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4209ec110, 0xc420824300, 0x100, 0x100, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc4209ee360)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).ReadByte(0xc4209ee360, 0xa, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:231 +0x51
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/lib/telnet.(*Conn).tryReadByte(0xc420813180, 0x45000a, 0x0, 0x0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/lib/telnet/conn.go:179 +0x33
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/lib/telnet.(*Conn).ReadByte(0xc420813180, 0xa, 0x0, 0x0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/lib/telnet/conn.go:210 +0x40
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/lib/telnet.(*Conn).Read(0xc420813180, 0xc42081528b, 0x1, 0x1, 0x1, 0x0, 0x0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/lib/telnet/conn.go:248 +0x40
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.TtyLiner(0x1f3e860, 0xc420813180, 0xc4209ee3c0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_console.go:111 +0xb8
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.watchVmConsole
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_console.go:78 +0x564

goroutine 116 [select]:
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*Fanout).start.func1(0xc420b52c80)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/fanout.go:84 +0x2fe
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*Fanout).start
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/fanout.go:109 +0x3f

goroutine 137 [chan receive]:
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.handleMsgToHyperstart(0xc420b86480, 0x7fc42b45b290, 0xc4209ec108)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:236 +0x108
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.handleCtlSock
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:182 +0xef

goroutine 131 [chan receive]:
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu.qmpHandler(0xc420b82b00)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu/qmp_handler.go:381 +0x3c7
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu.(*QemuContext).Launch
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu/qemu.go:154 +0x6b

goroutine 135 [sleep]:
time.Sleep(0x3b9aca00)
    /usr/local/go/src/runtime/time.go:59 +0xe1
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu.(*QemuLogFile).Read(0xc420b47c60, 0xc4207f9000, 0x1000, 0x1000, 0x1f72478, 0xc4206f2d98, 0x4b7246)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu/qemu_process.go:40 +0x1ac
bufio.(*Reader).fill(0xc4206f2ee8)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).ReadSlice(0xc4206f2ee8, 0x1f7240a, 0xc400000000, 0x1, 0xc4206f2ea8, 0x2, 0x2)
    /usr/local/go/src/bufio/bufio.go:330 +0xb5
bufio.(*Reader).ReadLine(0xc4206f2ee8, 0x2, 0x2, 0x1301b20, 0xc420801a30, 0x5a, 0x0)
    /usr/local/go/src/bufio/bufio.go:359 +0x37
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu.(*QemuLogFile).Watch(0xc420b47c60)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu/qemu_process.go:65 +0xb8
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu.launchQemu
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu/qemu_process.go:213 +0xa6d

goroutine 136 [chan receive]:
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu.watchDog(0xc420b86360, 0xc420b861e0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu/qemu_process.go:82 +0x87
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu.(*QemuContext).watchPid
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/qemu/qemu_process.go:114 +0x83

goroutine 138 [IO wait]:
net.runtime_pollWait(0x7fc42b51e3e8, 0x72, 0x14)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4208193a0, 0x72, 0xc420a58cd0, 0xc4200101b0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4208193a0, 0x1f438e0, 0xc4200101b0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420819340, 0xc421288200, 0x8, 0x200, 0x0, 0x1f438e0, 0xc4200101b0)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4209ec108, 0xc421288200, 0x8, 0x200, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.readVmMessage(0x7fc419e66b68, 0xc4209ec108, 0xc420a58ed8, 0x3, 0x3)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:135 +0x10e
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.handleMsgFromHyperstart(0xc420b86480, 0x7fc419e66b68, 0xc4209ec108)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:335 +0xa3
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart.handleCtlSock
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hyperstart/libhyperstart/json.go:183 +0x14c

goroutine 140 [select]:
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*Vm).WaitResponse.func1(0xc420b52be0, 0xc420bb3920, 0xc420bb3860, 0x163fc68, 0xc420bb38c0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm.go:126 +0x2af
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*Vm).WaitResponse
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm.go:141 +0x118

goroutine 145 [semacquire]:
sync.runtime_Semacquire(0xc42119dc3c)
    /usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc42119dc30)
    /usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/hyperhq/hyperd/daemon/pod.(*Container).AttachStreams(0xc420a60600, 0x1000000, 0x0, 0x0, 0x1f47120, 0xc4200b5618, 0x1f3f820, 0xc4200b5628, 0x0, 0x0, ...)
    /root/work/gopath/src/github.com/hyperhq/hyperd/daemon/pod/container.go:1381 +0x41a
created by github.com/hyperhq/hyperd/daemon/pod.(*Container).startLogging
    /root/work/gopath/src/github.com/hyperhq/hyperd/daemon/pod/container.go:1033 +0x2c4

goroutine 162 [semacquire]:
sync.runtime_notifyListWait(0xc42068cdc0, 0x0)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42068cdb0)
    /usr/local/go/src/sync/cond.go:57 +0x80
io.(*pipe).read(0xc42068cd80, 0xc421226000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/io/pipe.go:47 +0x102
io.(*PipeReader).Read(0xc4200b5610, 0xc421226000, 0x1000, 0x1000, 0x0, 0xc4207aa4c0, 0x42672f)
    /usr/local/go/src/io/pipe.go:129 +0x4c
bufio.(*Reader).fill(0xc42112aec8)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).ReadSlice(0xc42112aec8, 0x7fc42b49290a, 0x1, 0x101000000000001, 0x41c1e1, 0xc42121c2bc, 0xc400000000)
    /usr/local/go/src/bufio/bufio.go:330 +0xb5
bufio.(*Reader).ReadBytes(0xc42112aec8, 0xa, 0xc42121c2a0, 0xc421226000, 0x1000, 0x1000, 0x0)
    /usr/local/go/src/bufio/bufio.go:408 +0x66
github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/logger.(*Copier).copySrc(0xc4212221c0, 0x156111b, 0x6, 0x1f3f7e0, 0xc4200b5610)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/logger/copier.go:53 +0x122
created by github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/logger.(*Copier).Run
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/logger/copier.go:40 +0x119

goroutine 163 [semacquire]:
sync.runtime_notifyListWait(0xc42068ce80, 0x0)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42068ce70)
    /usr/local/go/src/sync/cond.go:57 +0x80
io.(*pipe).read(0xc42068ce40, 0xc421227000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/io/pipe.go:47 +0x102
io.(*PipeReader).Read(0xc4200b5620, 0xc421227000, 0x1000, 0x1000, 0x55e873, 0x1f1fde8, 0x14)
    /usr/local/go/src/io/pipe.go:129 +0x4c
bufio.(*Reader).fill(0xc42112bec8)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).ReadSlice(0xc42112bec8, 0xa, 0x0, 0x10000c420a4c550, 0x41c1e1, 0xc42121c2bc, 0xc400000000)
    /usr/local/go/src/bufio/bufio.go:330 +0xb5
bufio.(*Reader).ReadBytes(0xc42112bec8, 0xa, 0xc42121c2a0, 0xc421227000, 0x1000, 0x1000, 0xc)
    /usr/local/go/src/bufio/bufio.go:408 +0x66
github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/logger.(*Copier).copySrc(0xc4212221c0, 0x1561115, 0x6, 0x1f3f7e0, 0xc4200b5620)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/logger/copier.go:53 +0x122
created by github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/logger.(*Copier).Run
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/docker/docker/daemon/logger/copier.go:40 +0x119

goroutine 164 [chan receive]:
github.com/hyperhq/hyperd/daemon/pod.(*Container).waitFinish(0xc420a60600, 0xffffffffffffffff)
    /root/work/gopath/src/github.com/hyperhq/hyperd/daemon/pod/container.go:1059 +0x113
created by github.com/hyperhq/hyperd/daemon/pod.(*Container).start
    /root/work/gopath/src/github.com/hyperhq/hyperd/daemon/pod/container.go:247 +0x258

goroutine 121 [chan receive]:
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*Vm).WaitProcess.func2(0xc420b86b40, 0xc420b86ae0)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm.go:221 +0x53
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.(*Vm).WaitProcess
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm.go:224 +0x1d8

goroutine 165 [semacquire]:
sync.runtime_notifyListWait(0xc420a3a4c0, 0x0)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc420a3a4b0)
    /usr/local/go/src/sync/cond.go:57 +0x80
io.(*pipe).read(0xc420a3a480, 0xc421240000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/io/pipe.go:47 +0x102
io.(*PipeReader).Read(0xc4201ad6b0, 0xc421240000, 0x8000, 0x8000, 0x8000, 0x8000, 0x1f95ba0)
    /usr/local/go/src/io/pipe.go:129 +0x4c
io.copyBuffer(0x7fc42b4f0c80, 0xc42121c310, 0x1f3f7e0, 0xc4201ad6b0, 0xc421240000, 0x8000, 0x8000, 0x13c4c40, 0x0, 0x1f3f7e0)
    /usr/local/go/src/io/io.go:390 +0x147
io.Copy(0x7fc42b4f0c80, 0xc42121c310, 0x1f3f7e0, 0xc4201ad6b0, 0xc4201ad6b0, 0x0, 0x0)
    /usr/local/go/src/io/io.go:360 +0x68
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.streamCopy.func2(0x1f45b20, 0xc42121c310, 0xc42118fc50, 0xc421207540, 0xc4211ebc50)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_states.go:186 +0xab
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.streamCopy
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_states.go:192 +0x2a5

goroutine 166 [semacquire]:
sync.runtime_notifyListWait(0xc42068d0c0, 0x0)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42068d0b0)
    /usr/local/go/src/sync/cond.go:57 +0x80
io.(*pipe).read(0xc42068d080, 0xc421248000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/io/pipe.go:47 +0x102
io.(*PipeReader).Read(0xc4200b5630, 0xc421248000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0)
    /usr/local/go/src/io/pipe.go:129 +0x4c
io.copyBuffer(0x7fc42b45b808, 0xc4201ad6c0, 0x1f3f7e0, 0xc4200b5630, 0xc421248000, 0x8000, 0x8000, 0x13c4c40, 0x0, 0x1f3f7e0)
    /usr/local/go/src/io/io.go:390 +0x147
io.Copy(0x7fc42b45b808, 0xc4201ad6c0, 0x1f3f7e0, 0xc4200b5630, 0xc4200b5630, 0x0, 0x0)
    /usr/local/go/src/io/io.go:360 +0x68
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.streamCopy.func3(0xc42118fc50, 0x1f470e0, 0xc4200b5630, 0xc421207540, 0xc4211ebc50, 0xc421207530)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_states.go:197 +0x74
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.streamCopy
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_states.go:202 +0x246

goroutine 167 [semacquire]:
sync.runtime_notifyListWait(0xc42068d180, 0x0)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42068d170)
    /usr/local/go/src/sync/cond.go:57 +0x80
io.(*pipe).read(0xc42068d140, 0xc421238000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/io/pipe.go:47 +0x102
io.(*PipeReader).Read(0xc4200b5640, 0xc421238000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0)
    /usr/local/go/src/io/pipe.go:129 +0x4c
io.copyBuffer(0x7fc42b45b808, 0xc4201ad6c8, 0x1f3f7e0, 0xc4200b5640, 0xc421238000, 0x8000, 0x8000, 0x13c4c40, 0x0, 0x1f3f7e0)
    /usr/local/go/src/io/io.go:390 +0x147
io.Copy(0x7fc42b45b808, 0xc4201ad6c8, 0x1f3f7e0, 0xc4200b5640, 0xc4200b5640, 0x0, 0x0)
    /usr/local/go/src/io/io.go:360 +0x68
github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.streamCopy.func4(0xc42118fc50, 0x1f470e0, 0xc4200b5640, 0xc421207540, 0xc4211ebc50, 0xc421207530)
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_states.go:207 +0x74
created by github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor.streamCopy
    /root/work/gopath/src/github.com/hyperhq/hyperd/vendor/github.com/hyperhq/runv/hypervisor/vm_states.go:212 +0x1c6
laijs commented 7 years ago

list(partial): watchHyperstart() WaitResponse.func1() watchVmConsole() qemu.qmpReceiver() newContainer.func1() -> libhyperstart.(jsonBasedHyperstart).WaitProcess() streamCopy.func1/2/3/4 vm.WaitProcess.func2 qemu.(QemuLogFile).Watch() hypervisor.(*Fanout).start.func1()