docker / machine

Machine management for a container-centric world
https://docs.docker.com/machine/
Apache License 2.0
6.63k stars 1.97k forks source link

Error from docker-machine on OS X (intermittent) #3813

Closed buckett closed 7 years ago

buckett commented 8 years ago

Sometimes when running a docker-machine command it blows up, I've got machines (virtualbox, generic and azure):

$ docker-machine active
fatal error: unexpected signal during runtime execution
[signal 0xb code=0x1 addr=0xb01dfacedebac1e pc=0x12e8b]

runtime stack:
runtime.throw(0x18c89c0, 0x2a)
    /usr/local/go/src/runtime/panic.go:547 +0x90
runtime.sigpanic()
    /usr/local/go/src/runtime/sigpanic_unix.go:12 +0x5a
runtime.unlock(0x22d24e0)
    /usr/local/go/src/runtime/lock_sema.go:107 +0x14b
runtime.(*mheap).alloc_m(0x22d24e0, 0x1, 0xb, 0xc800000000)
    /usr/local/go/src/runtime/mheap.go:492 +0x314
runtime.(*mheap).alloc.func1()
    /usr/local/go/src/runtime/mheap.go:502 +0x41
runtime.systemstack(0xc82029de58)
    /usr/local/go/src/runtime/asm_amd64.s:307 +0xab
runtime.(*mheap).alloc(0x22d24e0, 0x1, 0x1000000000b, 0x12b2f)
    /usr/local/go/src/runtime/mheap.go:503 +0x63
runtime.(*mcentral).grow(0x22d3c80, 0x0)
    /usr/local/go/src/runtime/mcentral.go:209 +0x93
runtime.(*mcentral).cacheSpan(0x22d3c80, 0x0)
    /usr/local/go/src/runtime/mcentral.go:89 +0x47d
runtime.(*mcache).refill(0x273d000, 0xb, 0xc82038eeb0)
    /usr/local/go/src/runtime/mcache.go:119 +0xcc
runtime.mallocgc.func2()
    /usr/local/go/src/runtime/malloc.go:642 +0x2b
runtime.systemstack(0xc820021500)
    /usr/local/go/src/runtime/asm_amd64.s:291 +0x79
runtime.mstart()
    /usr/local/go/src/runtime/proc.go:1051

goroutine 53 [running]:
runtime.systemstack_switch()
    /usr/local/go/src/runtime/asm_amd64.s:245 fp=0xc82038e970 sp=0xc82038e968
runtime.mallocgc(0xa0, 0xdd8c40, 0x1, 0xc8200c69c0)
    /usr/local/go/src/runtime/malloc.go:643 +0x869 fp=0xc82038ea48 sp=0xc82038e970
runtime.newarray(0xdd8c40, 0x13, 0xb0955d)
    /usr/local/go/src/runtime/malloc.go:798 +0xc9 fp=0xc82038ea88 sp=0xc82038ea48
runtime.makeslice(0x156cf80, 0x13, 0x13, 0x0, 0x0, 0x0)
    /usr/local/go/src/runtime/slice.go:32 +0x165 fp=0xc82038ead8 sp=0xc82038ea88
math/big.nat.divLarge(0xc8202353b0, 0x1, 0x5, 0xc8200c69c0, 0x14, 0x18, 0xc8200c6840, 0x13, 0x18, 0xc8200c6900, ...)
    /usr/local/go/src/math/big/nat.go:571 +0x4d6 fp=0xc82038ec88 sp=0xc82038ead8
math/big.nat.div(0xc8202353b0, 0x1, 0x5, 0x0, 0x0, 0x0, 0xc8200c6840, 0x13, 0x18, 0xc8200c6900, ...)
    /usr/local/go/src/math/big/nat.go:538 +0x2f5 fp=0xc82038ed30 sp=0xc82038ec88
math/big.(*Int).QuoRem(0xc82038ee88, 0xc82021a780, 0xc82021a7a0, 0xc82021a7c0, 0xc82038ee88, 0xc82021a7a0)
    /usr/local/go/src/math/big/int.go:227 +0xe6 fp=0xc82038edf8 sp=0xc82038ed30
math/big.(*Int).GCD(0xc82038efd0, 0xc82023be60, 0xc82038efb0, 0xc82023be40, 0xc820362880, 0xc82023be40)
    /usr/local/go/src/math/big/int.go:464 +0x2c6 fp=0xc82038ef70 sp=0xc82038edf8
crypto/rsa.modInverse(0xc82023be40, 0xc820362880, 0x0, 0xc82023be00)
    /usr/local/go/src/crypto/rsa/rsa.go:411 +0xb3 fp=0xc82038eff8 sp=0xc82038ef70
crypto/rsa.decrypt(0x27913b0, 0xc820010840, 0xc82036a120, 0xc82038f290, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/rsa/rsa.go:490 +0x1b4 fp=0xc82038f138 sp=0xc82038eff8
crypto/rsa.decryptAndCheck(0x27913b0, 0xc820010840, 0xc82036a120, 0xc82038f290, 0xc82038f290, 0x0, 0x0)
    /usr/local/go/src/crypto/rsa/rsa.go:542 +0x66 fp=0xc82038f1c0 sp=0xc82038f138
crypto/rsa.SignPKCS1v15(0x27913b0, 0xc820010840, 0xc82036a120, 0x5, 0xc82023be20, 0x20, 0x20, 0x0, 0x0, 0x0, ...)
    /usr/local/go/src/crypto/rsa/pkcs1v15.go:249 +0x3d1 fp=0xc82038f2b8 sp=0xc82038f1c0
crypto/rsa.(*PrivateKey).Sign(0xc82036a120, 0x27913b0, 0xc820010840, 0xc82023be20, 0x20, 0x20, 0x27d53e0, 0xc82022eaa0, 0x0, 0x0, ...)
    /usr/local/go/src/crypto/rsa/rsa.go:102 +0x184 fp=0xc82038f328 sp=0xc82038f2b8
crypto/tls.(*clientHandshakeState).doFullHandshake(0xc82038fe48, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/handshake_client.go:473 +0x1a0a fp=0xc82038f9e0 sp=0xc82038f328
crypto/tls.(*Conn).clientHandshake(0xc8203ea000, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/handshake_client.go:213 +0x1737 fp=0xc82038fef8 sp=0xc82038f9e0
crypto/tls.(*Conn).Handshake(0xc8203ea000, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1033 +0xec fp=0xc82038ff38 sp=0xc82038fef8
net/http.(*Transport).dialConn.func3(0xc8203ea000, 0x0, 0xc8202b89c0)
    /usr/local/go/src/net/http/transport.go:828 +0x21 fp=0xc82038ff78 sp=0xc82038ff38
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc82038ff80 sp=0xc82038ff78
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:833 +0x986

goroutine 1 [chan receive]:
github.com/docker/machine/commands.getHostListItems(0xc8202beda0, 0x3, 0x4, 0xc820218330, 0x2540be400, 0x0, 0x0, 0x0)
    /go/src/github.com/docker/machine/commands/ls.go:453 +0x2b1
github.com/docker/machine/commands.cmdActive(0x27922c0, 0xc8200341e0, 0x2792340, 0xc8201668c0, 0x0, 0x0)
    /go/src/github.com/docker/machine/commands/active.go:34 +0x29e
github.com/docker/machine/commands.runCommand.func1(0xc8202110e0)
    /go/src/github.com/docker/machine/commands/commands.go:163 +0x5e3
github.com/docker/machine/vendor/github.com/codegangsta/cli.Command.Run(0x158d8f8, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1771d40, 0x1d, 0x0, ...)
    /go/src/github.com/docker/machine/vendor/github.com/codegangsta/cli/command.go:137 +0x1081
github.com/docker/machine/vendor/github.com/codegangsta/cli.(*App).Run(0xc820210d80, 0xc82000a400, 0x2, 0x2, 0x0, 0x0)
    /go/src/github.com/docker/machine/vendor/github.com/codegangsta/cli/app.go:175 +0xffa
main.main()
    /go/src/github.com/docker/machine/cmd/machine.go:163 +0xcc0

goroutine 5 [syscall]:
os/signal.signal_recv(0x0)
    /usr/local/go/src/runtime/sigqueue.go:116 +0x132
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
    /usr/local/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 7 [select]:
github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).execServer(0xc820166af0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:206 +0x9a3
github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).Serve(0xc820166af0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:221 +0x2d
github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver.func1(0xc820166af0)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:130 +0x25
created by github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:135 +0xcf

goroutine 17 [syscall]:
syscall.Syscall(0x3, 0x3, 0xc82024e010, 0xff0, 0x0, 0x9f, 0x1a48613)
    /usr/local/go/src/syscall/asm_darwin_amd64.s:16 +0x5
syscall.read(0x3, 0xc82024e010, 0xff0, 0xff0, 0xc81ffff80d, 0x0, 0x0)
    /usr/local/go/src/syscall/zsyscall_darwin_amd64.go:972 +0x5f
syscall.Read(0x3, 0xc82024e010, 0xff0, 0xff0, 0x40, 0x0, 0x0)
    /usr/local/go/src/syscall/syscall_unix.go:161 +0x4d
os.(*File).read(0xc820238000, 0xc82024e010, 0xff0, 0xff0, 0x14a5e7, 0x0, 0x0)
    /usr/local/go/src/os/file_unix.go:228 +0x75
os.(*File).Read(0xc820238000, 0xc82024e010, 0xff0, 0xff0, 0x0, 0x0, 0x0)
    /usr/local/go/src/os/file.go:95 +0x8a
bufio.(*Scanner).Scan(0xc82023c000, 0xc820032778)
    /usr/local/go/src/bufio/scan.go:208 +0x97b
github.com/docker/machine/libmachine/drivers/plugin/localbinary.stream(0xc82023c000, 0xc8202440c0)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:171 +0x30
created by github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).AttachStream
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:182 +0x5d

goroutine 18 [syscall]:
syscall.Syscall(0x3, 0x5, 0xc82024f2ad, 0xd53, 0x68, 0x0, 0x0)
    /usr/local/go/src/syscall/asm_darwin_amd64.s:16 +0x5
syscall.read(0x5, 0xc82024f2ad, 0xd53, 0xd53, 0x68, 0x0, 0x0)
    /usr/local/go/src/syscall/zsyscall_darwin_amd64.go:972 +0x5f
syscall.Read(0x5, 0xc82024f2ad, 0xd53, 0xd53, 0x68, 0x0, 0x0)
    /usr/local/go/src/syscall/syscall_unix.go:161 +0x4d
os.(*File).read(0xc820238010, 0xc82024f2ad, 0xd53, 0xd53, 0x1497f, 0x0, 0x0)
    /usr/local/go/src/os/file_unix.go:228 +0x75
os.(*File).Read(0xc820238010, 0xc82024f2ad, 0xd53, 0xd53, 0x0, 0x0, 0x0)
    /usr/local/go/src/os/file.go:95 +0x8a
bufio.(*Scanner).Scan(0xc82023c080, 0xc820244120)
    /usr/local/go/src/bufio/scan.go:208 +0x97b
github.com/docker/machine/libmachine/drivers/plugin/localbinary.stream(0xc82023c080, 0xc820244120)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:171 +0x30
created by github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).AttachStream
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:182 +0x5d

goroutine 33 [IO wait]:
net.runtime_pollWait(0x280e000, 0x72, 0xc820270000)
    /usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc820266060, 0x72, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc820266060, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc820266000, 0xc820270000, 0x1000, 0x1000, 0x0, 0x278d050, 0xc820012068)
    /usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc820268000, 0xc820270000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:172 +0xe4
bufio.(*Reader).fill(0xc8202600c0)
    /usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Read(0xc8202600c0, 0xc820250060, 0x1, 0x9, 0x10, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:207 +0x260
io.ReadAtLeast(0x280e138, 0xc8202600c0, 0xc820250060, 0x1, 0x9, 0x1, 0x0, 0x0, 0x0)
    /usr/local/go/src/io/io.go:297 +0xe6
io.ReadFull(0x280e138, 0xc8202600c0, 0xc820250060, 0x1, 0x9, 0xc82026e070, 0x0, 0x0)
    /usr/local/go/src/io/io.go:315 +0x62
encoding/gob.decodeUintReader(0x280e138, 0xc8202600c0, 0xc820250060, 0x9, 0x9, 0x0, 0x1, 0x0, 0x0)
    /usr/local/go/src/encoding/gob/decode.go:119 +0x92
encoding/gob.(*Decoder).recvMessage(0xc82026e000, 0xc820127bc0)
    /usr/local/go/src/encoding/gob/decoder.go:76 +0x5e
encoding/gob.(*Decoder).decodeTypeSequence(0xc82026e000, 0x1a4c600, 0xc82026e000)
    /usr/local/go/src/encoding/gob/decoder.go:140 +0x47
encoding/gob.(*Decoder).DecodeValue(0xc82026e000, 0xd90b00, 0xc82029e000, 0x16, 0x0, 0x0)
    /usr/local/go/src/encoding/gob/decoder.go:208 +0x15d
encoding/gob.(*Decoder).Decode(0xc82026e000, 0xd90b00, 0xc82029e000, 0x0, 0x0)
    /usr/local/go/src/encoding/gob/decoder.go:185 +0x289
net/rpc.(*gobClientCodec).ReadResponseHeader(0xc82025e1e0, 0xc82029e000, 0x0, 0x0)
    /usr/local/go/src/net/rpc/client.go:223 +0x51
net/rpc.(*Client).input(0xc820260180)
    /usr/local/go/src/net/rpc/client.go:109 +0xbf
created by net/rpc.NewClientWithCodec
    /usr/local/go/src/net/rpc/client.go:201 +0xd7

goroutine 49 [select]:
github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver.func2(0xc820262120)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:175 +0x375
created by github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:187 +0xae1

goroutine 65 [select]:
github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).execServer(0xc8202cc280, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:206 +0x9a3
github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).Serve(0xc8202cc280, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:221 +0x2d
github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver.func1(0xc8202cc280)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:130 +0x25
created by github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:135 +0xcf

goroutine 66 [syscall]:
syscall.Syscall(0x3, 0x7, 0xc8202f0010, 0xff0, 0x0, 0x0, 0x0)
    /usr/local/go/src/syscall/asm_darwin_amd64.s:16 +0x5
syscall.read(0x7, 0xc8202f0010, 0xff0, 0xff0, 0x0, 0x0, 0x0)
    /usr/local/go/src/syscall/zsyscall_darwin_amd64.go:972 +0x5f
syscall.Read(0x7, 0xc8202f0010, 0xff0, 0xff0, 0x0, 0x0, 0x0)
    /usr/local/go/src/syscall/syscall_unix.go:161 +0x4d
os.(*File).read(0xc8202c2050, 0xc8202f0010, 0xff0, 0xff0, 0x14a5e7, 0x0, 0x0)
    /usr/local/go/src/os/file_unix.go:228 +0x75
os.(*File).Read(0xc8202c2050, 0xc8202f0010, 0xff0, 0xff0, 0x0, 0x0, 0x0)
    /usr/local/go/src/os/file.go:95 +0x8a
bufio.(*Scanner).Scan(0xc8202dc080, 0x0)
    /usr/local/go/src/bufio/scan.go:208 +0x97b
github.com/docker/machine/libmachine/drivers/plugin/localbinary.stream(0xc8202dc080, 0xc8202ca2a0)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:171 +0x30
created by github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).AttachStream
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:182 +0x5d

goroutine 67 [syscall]:
syscall.Syscall(0x3, 0x9, 0xc8202f1048, 0xfb8, 0x48, 0x0, 0x0)
    /usr/local/go/src/syscall/asm_darwin_amd64.s:16 +0x5
syscall.read(0x9, 0xc8202f1048, 0xfb8, 0xfb8, 0x48, 0x0, 0x0)
    /usr/local/go/src/syscall/zsyscall_darwin_amd64.go:972 +0x5f
syscall.Read(0x9, 0xc8202f1048, 0xfb8, 0xfb8, 0x48, 0x0, 0x0)
    /usr/local/go/src/syscall/syscall_unix.go:161 +0x4d
os.(*File).read(0xc8202c2060, 0xc8202f1048, 0xfb8, 0xfb8, 0x1497f, 0x0, 0x0)
    /usr/local/go/src/os/file_unix.go:228 +0x75
os.(*File).Read(0xc8202c2060, 0xc8202f1048, 0xfb8, 0xfb8, 0x0, 0x0, 0x0)
    /usr/local/go/src/os/file.go:95 +0x8a
bufio.(*Scanner).Scan(0xc8202dc100, 0xc8202ca300)
    /usr/local/go/src/bufio/scan.go:208 +0x97b
github.com/docker/machine/libmachine/drivers/plugin/localbinary.stream(0xc8202dc100, 0xc8202ca300)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:171 +0x30
created by github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).AttachStream
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:182 +0x5d

goroutine 19 [IO wait]:
net.runtime_pollWait(0x280df40, 0x72, 0xc820306000)
    /usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc82024c0d0, 0x72, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc82024c0d0, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc82024c070, 0xc820306000, 0x1000, 0x1000, 0x0, 0x278d050, 0xc820012068)
    /usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc820238030, 0xc820306000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:172 +0xe4
bufio.(*Reader).fill(0xc820244240)
    /usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Read(0xc820244240, 0xc82022e170, 0x1, 0x9, 0x10, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:207 +0x260
io.ReadAtLeast(0x280e138, 0xc820244240, 0xc82022e170, 0x1, 0x9, 0x1, 0x0, 0x0, 0x0)
    /usr/local/go/src/io/io.go:297 +0xe6
io.ReadFull(0x280e138, 0xc820244240, 0xc82022e170, 0x1, 0x9, 0xc82023c1f0, 0x0, 0x0)
    /usr/local/go/src/io/io.go:315 +0x62
encoding/gob.decodeUintReader(0x280e138, 0xc820244240, 0xc82022e170, 0x9, 0x9, 0x0, 0x1, 0x0, 0x0)
    /usr/local/go/src/encoding/gob/decode.go:119 +0x92
encoding/gob.(*Decoder).recvMessage(0xc82023c180, 0xc820313bc0)
    /usr/local/go/src/encoding/gob/decoder.go:76 +0x5e
encoding/gob.(*Decoder).decodeTypeSequence(0xc82023c180, 0x1a4c600, 0xc82023c180)
    /usr/local/go/src/encoding/gob/decoder.go:140 +0x47
encoding/gob.(*Decoder).DecodeValue(0xc82023c180, 0xd90b00, 0xc8202344e0, 0x16, 0x0, 0x0)
    /usr/local/go/src/encoding/gob/decoder.go:208 +0x15d
encoding/gob.(*Decoder).Decode(0xc82023c180, 0xd90b00, 0xc8202344e0, 0x0, 0x0)
    /usr/local/go/src/encoding/gob/decoder.go:185 +0x289
net/rpc.(*gobClientCodec).ReadResponseHeader(0xc820234450, 0xc8202344e0, 0x0, 0x0)
    /usr/local/go/src/net/rpc/client.go:223 +0x51
net/rpc.(*Client).input(0xc820244300)
    /usr/local/go/src/net/rpc/client.go:109 +0xbf
created by net/rpc.NewClientWithCodec
    /usr/local/go/src/net/rpc/client.go:201 +0xd7

goroutine 50 [select]:
github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver.func2(0xc82023a6c0)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:175 +0x375
created by github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:187 +0xae1

goroutine 20 [select]:
github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).execServer(0xc8202424b0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:206 +0x9a3
github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).Serve(0xc8202424b0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:221 +0x2d
github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver.func1(0xc8202424b0)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:130 +0x25
created by github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:135 +0xcf

goroutine 21 [syscall]:
syscall.Syscall(0x3, 0xa, 0xc820307010, 0xff0, 0xac835d, 0xc82023c180, 0x1a4c600)
    /usr/local/go/src/syscall/asm_darwin_amd64.s:16 +0x5
syscall.read(0xa, 0xc820307010, 0xff0, 0xff0, 0x1a485c6, 0x0, 0x0)
    /usr/local/go/src/syscall/zsyscall_darwin_amd64.go:972 +0x5f
syscall.Read(0xa, 0xc820307010, 0xff0, 0xff0, 0xd90b00, 0x0, 0x0)
    /usr/local/go/src/syscall/syscall_unix.go:161 +0x4d
os.(*File).read(0xc820238088, 0xc820307010, 0xff0, 0xff0, 0x14a5e7, 0x0, 0x0)
    /usr/local/go/src/os/file_unix.go:228 +0x75
os.(*File).Read(0xc820238088, 0xc820307010, 0xff0, 0xff0, 0xd90b00, 0x0, 0x0)
    /usr/local/go/src/os/file.go:95 +0x8a
bufio.(*Scanner).Scan(0xc82023c280, 0x0)
    /usr/local/go/src/bufio/scan.go:208 +0x97b
github.com/docker/machine/libmachine/drivers/plugin/localbinary.stream(0xc82023c280, 0xc8202446c0)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:171 +0x30
created by github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).AttachStream
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:182 +0x5d

goroutine 22 [syscall]:
syscall.Syscall(0x3, 0xc, 0xc820324000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/syscall/asm_darwin_amd64.s:16 +0x5
syscall.read(0xc, 0xc820324000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/syscall/zsyscall_darwin_amd64.go:972 +0x5f
syscall.Read(0xc, 0xc820324000, 0x1000, 0x1000, 0xc820324000, 0x0, 0x0)
    /usr/local/go/src/syscall/syscall_unix.go:161 +0x4d
os.(*File).read(0xc820238098, 0xc820324000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/os/file_unix.go:228 +0x75
os.(*File).Read(0xc820238098, 0xc820324000, 0x1000, 0x1000, 0x1000, 0x0, 0x0)
    /usr/local/go/src/os/file.go:95 +0x8a
bufio.(*Scanner).Scan(0xc82023c300, 0x0)
    /usr/local/go/src/bufio/scan.go:208 +0x97b
github.com/docker/machine/libmachine/drivers/plugin/localbinary.stream(0xc82023c300, 0xc820244720)
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:171 +0x30
created by github.com/docker/machine/libmachine/drivers/plugin/localbinary.(*Plugin).AttachStream
    /go/src/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go:182 +0x5d

goroutine 68 [IO wait]:
net.runtime_pollWait(0x280de80, 0x72, 0xc820344000)
    /usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc8202c41b0, 0x72, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8202c41b0, 0x0, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8202c4150, 0xc820344000, 0x1000, 0x1000, 0x0, 0x278d050, 0xc820012068)
    /usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc8202c2080, 0xc820344000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:172 +0xe4
bufio.(*Reader).fill(0xc8202ca3c0)
    /usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Read(0xc8202ca3c0, 0xc8202c62d0, 0x1, 0x9, 0x10, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:207 +0x260
io.ReadAtLeast(0x280e138, 0xc8202ca3c0, 0xc8202c62d0, 0x1, 0x9, 0x1, 0x0, 0x0, 0x0)
    /usr/local/go/src/io/io.go:297 +0xe6
io.ReadFull(0x280e138, 0xc8202ca3c0, 0xc8202c62d0, 0x1, 0x9, 0xc8202dc1f0, 0x0, 0x0)
    /usr/local/go/src/io/io.go:315 +0x62
encoding/gob.decodeUintReader(0x280e138, 0xc8202ca3c0, 0xc8202c62d0, 0x9, 0x9, 0x0, 0x1, 0x0, 0x0)
    /usr/local/go/src/encoding/gob/decode.go:119 +0x92
encoding/gob.(*Decoder).recvMessage(0xc8202dc180, 0xc82030fbc0)
    /usr/local/go/src/encoding/gob/decoder.go:76 +0x5e
encoding/gob.(*Decoder).decodeTypeSequence(0xc8202dc180, 0x1a4c600, 0xc8202dc180)
    /usr/local/go/src/encoding/gob/decoder.go:140 +0x47
encoding/gob.(*Decoder).DecodeValue(0xc8202dc180, 0xd90b00, 0xc8202c0cf0, 0x16, 0x0, 0x0)
    /usr/local/go/src/encoding/gob/decoder.go:208 +0x15d
encoding/gob.(*Decoder).Decode(0xc8202dc180, 0xd90b00, 0xc8202c0cf0, 0x0, 0x0)
    /usr/local/go/src/encoding/gob/decoder.go:185 +0x289
net/rpc.(*gobClientCodec).ReadResponseHeader(0xc8202c0c60, 0xc8202c0cf0, 0x0, 0x0)
    /usr/local/go/src/net/rpc/client.go:223 +0x51
net/rpc.(*Client).input(0xc8202ca480)
    /usr/local/go/src/net/rpc/client.go:109 +0xbf
created by net/rpc.NewClientWithCodec
    /usr/local/go/src/net/rpc/client.go:201 +0xd7

goroutine 51 [select]:
github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver.func2(0xc8202bebe0)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:175 +0x375
created by github.com/docker/machine/libmachine/drivers/rpc.(*DefaultRPCClientDriverFactory).NewRPCClientDriver
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:187 +0xae1

goroutine 69 [select]:
github.com/docker/machine/commands.getHostState(0xc820016c60, 0xc8202ca600, 0x2540be400)
    /go/src/github.com/docker/machine/commands/ls.go:425 +0x28e
created by github.com/docker/machine/commands.getHostListItems
    /go/src/github.com/docker/machine/commands/ls.go:449 +0x220

goroutine 70 [select]:
github.com/docker/machine/commands.getHostState(0xc8202ca060, 0xc8202ca600, 0x2540be400)
    /go/src/github.com/docker/machine/commands/ls.go:425 +0x28e
created by github.com/docker/machine/commands.getHostListItems
    /go/src/github.com/docker/machine/commands/ls.go:449 +0x220

goroutine 71 [select]:
github.com/docker/machine/commands.getHostState(0xc820244480, 0xc8202ca600, 0x2540be400)
    /go/src/github.com/docker/machine/commands/ls.go:425 +0x28e
created by github.com/docker/machine/commands.getHostListItems
    /go/src/github.com/docker/machine/commands/ls.go:449 +0x220

goroutine 72 [select]:
net/http.(*Transport).getConn(0xc82036e0c0, 0xc8203c80e0, 0x0, 0xc8203c4ae0, 0x5, 0xc8203c4ae8, 0x18, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/http/transport.go:711 +0x4ef
net/http.(*Transport).RoundTrip(0xc82036e0c0, 0xc8203c80e0, 0xc82036e0c0, 0x0, 0x0)
    /usr/local/go/src/net/http/transport.go:311 +0x7e9
net/http.send(0xc8203c80e0, 0x27915a8, 0xc82036e0c0, 0x0, 0x0, 0x0, 0xc8203c4b40, 0x0, 0x0)
    /usr/local/go/src/net/http/client.go:260 +0x6b7
net/http.(*Client).send(0xc8203c4a80, 0xc8203c80e0, 0x0, 0x0, 0x0, 0x147b7, 0x0, 0x0)
    /usr/local/go/src/net/http/client.go:155 +0x185
net/http.(*Client).doFollowingRedirects(0xc8203c4a80, 0xc8203c80e0, 0x1a4c148, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/http/client.go:475 +0x8a4
net/http.(*Client).Do(0xc8203c4a80, 0xc8203c80e0, 0xc, 0x0, 0x0)
    /usr/local/go/src/net/http/client.go:188 +0xff
github.com/docker/machine/vendor/github.com/samalba/dockerclient.(*DockerClient).doStreamRequest(0xc8203c4ab0, 0x158a1b0, 0x3, 0xc820364f30, 0xe, 0x28d1058, 0xc8203ba7e0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/docker/machine/vendor/github.com/samalba/dockerclient/dockerclient.go:99 +0x3af
github.com/docker/machine/vendor/github.com/samalba/dockerclient.(*DockerClient).doRequest(0xc8203c4ab0, 0x158a1b0, 0x3, 0xc820364f30, 0xe, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/docker/machine/vendor/github.com/samalba/dockerclient/dockerclient.go:72 +0x166
github.com/docker/machine/vendor/github.com/samalba/dockerclient.(*DockerClient).Version(0xc8203c4ab0, 0xc820362080, 0x0, 0x0)
    /go/src/github.com/docker/machine/vendor/github.com/samalba/dockerclient/dockerclient.go:567 +0x18d
github.com/docker/machine/libmachine/mcndockerclient.(*defaultDockerVersioner).DockerVersion(0x22eb190, 0x28cd000, 0xc820362080, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/mcndockerclient/docker_versioner.go:23 +0x177
github.com/docker/machine/libmachine/mcndockerclient.DockerVersion(0x28cd000, 0xc820362080, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/mcndockerclient/docker_versioner.go:12 +0x57
github.com/docker/machine/commands.attemptGetHostState(0xc820244480, 0xc8202ca660)
    /go/src/github.com/docker/machine/commands/ls.go:362 +0x1c3
created by github.com/docker/machine/commands.getHostState
    /go/src/github.com/docker/machine/commands/ls.go:423 +0x6f

goroutine 52 [chan receive]:
net/rpc.(*Client).Call(0xc820260180, 0xc8202b14e0, 0x16, 0xdc8060, 0x22eb190, 0xd942e0, 0xc8202b4900, 0x0, 0x0)
    /usr/local/go/src/net/rpc/client.go:315 +0xcd
github.com/docker/machine/libmachine/drivers/rpc.(*InternalClient).Call(0xc82025e240, 0x1588a00, 0x7, 0xdc8060, 0x22eb190, 0xd942e0, 0xc8202b4900, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:92 +0x128
github.com/docker/machine/libmachine/drivers/rpc.(*RPCClientDriver).rpcStringCall(0xc820262120, 0x1588a00, 0x7, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:230 +0xe0
github.com/docker/machine/libmachine/drivers/rpc.(*RPCClientDriver).GetURL(0xc820262120, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:276 +0x4e
github.com/docker/machine/libmachine/host.(*Host).URL(0xc820016c60, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/host/host.go:192 +0x52
github.com/docker/machine/commands.attemptGetHostState(0xc820016c60, 0xc8202b87e0)
    /go/src/github.com/docker/machine/commands/ls.go:341 +0xa0
created by github.com/docker/machine/commands.getHostState
    /go/src/github.com/docker/machine/commands/ls.go:423 +0x6f

goroutine 34 [chan receive]:
net/rpc.(*Client).Call(0xc820244300, 0xc8202623c0, 0x16, 0xdc8060, 0x22eb190, 0xd942e0, 0xc820250280, 0x0, 0x0)
    /usr/local/go/src/net/rpc/client.go:315 +0xcd
github.com/docker/machine/libmachine/drivers/rpc.(*InternalClient).Call(0xc8202344b0, 0x1588a00, 0x7, 0xdc8060, 0x22eb190, 0xd942e0, 0xc820250280, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:92 +0x128
github.com/docker/machine/libmachine/drivers/rpc.(*RPCClientDriver).rpcStringCall(0xc82023a6c0, 0x1588a00, 0x7, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:230 +0xe0
github.com/docker/machine/libmachine/drivers/rpc.(*RPCClientDriver).GetURL(0xc82023a6c0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/client_driver.go:276 +0x4e
github.com/docker/machine/libmachine/drivers.(*SerialDriver).GetURL(0xc82023a860, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/drivers/serial.go:109 +0xd3
github.com/docker/machine/libmachine/host.(*Host).URL(0xc8202ca060, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/docker/machine/libmachine/host/host.go:192 +0x52
github.com/docker/machine/commands.attemptGetHostState(0xc8202ca060, 0xc820260600)
    /go/src/github.com/docker/machine/commands/ls.go:341 +0xa0
created by github.com/docker/machine/commands.getHostState
    /go/src/github.com/docker/machine/commands/ls.go:423 +0x6f

goroutine 81 [chan receive]:
net/http.(*Transport).dialConn(0xc82036e0c0, 0x0, 0xc8203c4ae0, 0x5, 0xc8203c4ae8, 0x18, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/http/transport.go:834 +0x9c2
net/http.(*Transport).getConn.func4(0xc82036e0c0, 0x0, 0xc8203c4ae0, 0x5, 0xc8203c4ae8, 0x18, 0xc8203bd800)
    /usr/local/go/src/net/http/transport.go:706 +0x66
created by net/http.(*Transport).getConn
    /usr/local/go/src/net/http/transport.go:708 +0x262
nathanleclaire commented 8 years ago

which machine version is this?

ahmetb commented 7 years ago

@buckett is there any possibility that you were resizing your terminal as docker-machine was running? https://github.com/golang/go/issues/16468

buckett commented 7 years ago

@nathanleclaire docker-machine 0.8.1

@ahmetalpbalkan I don't think so, I'm not able to reliably reproduce it, although when running dm status in a loop I did just get another stack trace (docker-machine 0.8.1 and macOS 10.12.1):

fatal error: unexpected signal during runtime execution
[signal 0xb code=0x1 addr=0xb01dfacedebac1e pc=0x12e8b]

runtime stack:
runtime.throw(0x18c89c0, 0x2a)
    /usr/local/go/src/runtime/panic.go:547 +0x90
runtime.sigpanic()
    /usr/local/go/src/runtime/sigpanic_unix.go:12 +0x5a
runtime.unlock(0x22d24e0)
    /usr/local/go/src/runtime/lock_sema.go:107 +0x14b
runtime.(*mheap).alloc_m(0x22d24e0, 0xb, 0x100000000, 0x22d3cf0)
    /usr/local/go/src/runtime/mheap.go:492 +0x314
runtime.(*mheap).alloc.func1()
    /usr/local/go/src/runtime/mheap.go:502 +0x41
runtime.systemstack(0x7fff5fbfece0)
    /usr/local/go/src/runtime/asm_amd64.s:307 +0xab
runtime.(*mheap).alloc(0x22d24e0, 0xb, 0x10100000000, 0x1883c)
    /usr/local/go/src/runtime/mheap.go:503 +0x63
runtime.largeAlloc(0x16000, 0x0, 0xc)
    /usr/local/go/src/runtime/malloc.go:766 +0xb3
runtime.mallocgc.func3()
    /usr/local/go/src/runtime/malloc.go:664 +0x33
runtime.systemstack(0x22cd500)
    /usr/local/go/src/runtime/asm_amd64.s:291 +0x79
runtime.mstart()
    /usr/local/go/src/runtime/proc.go:1051

goroutine 1 [running, locked to thread]:
runtime.systemstack_switch()
    /usr/local/go/src/runtime/asm_amd64.s:245 fp=0xc820041970 sp=0xc820041968
runtime.mallocgc(0x16000, 0x132f260, 0x0, 0x6c0c)
    /usr/local/go/src/runtime/malloc.go:665 +0x9eb fp=0xc820041a48 sp=0xc820041970
runtime.newarray(0x132f260, 0x200, 0x4)
    /usr/local/go/src/runtime/malloc.go:798 +0xc9 fp=0xc820041a88 sp=0xc820041a48
runtime.hashGrow(0xdd7fc0, 0xc8200d21e0)
    /usr/local/go/src/runtime/hashmap.go:809 +0x82 fp=0xc820041ab8 sp=0xc820041a88
runtime.mapassign1(0xdd7fc0, 0xc8200d21e0, 0x1a8b780, 0x1a8b790)
    /usr/local/go/src/runtime/hashmap.go:509 +0x5d2 fp=0xc820041b60 sp=0xc820041ab8
html.init()
    /usr/local/go/src/html/entity.go:2154 +0xe8 fp=0xc820041bc8 sp=0xc820041b60
html/template.init()
    /usr/local/go/src/html/template/url.go:105 +0x70 fp=0xc820041cc8 sp=0xc820041bc8
net/rpc.init()
    /usr/local/go/src/net/rpc/server.go:712 +0x6f fp=0xc820041d30 sp=0xc820041cc8
github.com/docker/machine/libmachine/drivers/rpc.init()
    /go/src/github.com/docker/machine/libmachine/drivers/rpc/server_driver.go:227 +0x4c fp=0xc820041d70 sp=0xc820041d30
github.com/docker/machine/libmachine.init()
    /go/src/github.com/docker/machine/libmachine/libmachine.go:188 +0x63 fp=0xc820041d78 sp=0xc820041d70
github.com/docker/machine/commands.init()
    /go/src/github.com/docker/machine/commands/version.go:40 +0x5f fp=0xc820041f48 sp=0xc820041d78
main.init()
    /go/src/github.com/docker/machine/cmd/machine.go:215 +0x58 fp=0xc820041f50 sp=0xc820041f48
runtime.main()
    /usr/local/go/src/runtime/proc.go:177 +0x27f fp=0xc820041fa0 sp=0xc820041f50
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc820041fa8 sp=0xc820041fa0

goroutine 5 [syscall]:
os/signal.signal_recv(0x0)
    /usr/local/go/src/runtime/sigqueue.go:116 +0x132
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
    /usr/local/go/src/os/signal/signal_unix.go:28 +0x37
Segmentation fault: 11
buckett commented 7 years ago

To get this error I was running while true; do docker-machine status default; done and can reliably get this error after a short while strangely while the loop is running I'll also get other errors without a stack trace along the lines of:

dial tcp: lookup ?e??W?1: invalid domain name
Segmentation fault: 11
Segmentation fault: 11
nathanleclaire commented 7 years ago

@buckett Segfault? Smells like cgo. Did you install with Homebrew? Can you retry with the pre-compiled binary from https://github.com/docker/machine/releases?

buckett commented 7 years ago

Precompiled binary https://github.com/docker/machine/releases

$ shasum -a 256 /usr/local/bin/docker-machine
28e07e42024d1f9b94e8d5953751556afe24cdfdc57d14e803506c3af47fe94b  /usr/local/bin/docker-machine

which matches the 0.8.1 release from there.

nathanleclaire commented 7 years ago

Cool, thanks. Mind trying the most recent version (0.9.0-rc2) to see if it fixes the problem? My tentative guess is that it's related to cgo code doing hostname lookups and that it might be resolved in slightly more recent Go versions. To see the panic with some preceding docker-machine --debug info would be useful as well.

buckett commented 7 years ago

Thanks, @nathanleclaire 0.9.0-rc2 works fine.

nathanleclaire commented 7 years ago

cool I'm going to go ahead and close then