search

AdRoll / hologram

Easy, painless AWS credentials on developer laptops.
Apache License 2.0
803 stars 42 forks source link

Sierra issues #81

Open copumpkin opened 7 years ago

copumpkin commented 7 years ago

79 upgraded to Go 1.7.1, but that doesn't seem sufficient to fix our issues. It seemed to work at first, but after a while, I get stack traces in the agent along the lines of:

failed MSpanList_Insert 0x8a48c8 0x2f9cbb4ded2cb 0x0
fatal error: MSpanList_Insert

runtime stack:
runtime.MSpanList_Insert(0x7e7bf8, 0x8a48c8)
    /usr/src/go/src/runtime/mheap.c:692 +0x8f
runtime.MHeap_Alloc(0x7e7640, 0x1, 0x10000000025, 0xe8e9)
    /usr/src/go/src/runtime/mheap.c:240 +0x66
runtime.MCentral_CacheSpan(0x7f0918, 0x0)
    /usr/src/go/src/runtime/mcentral.c:85 +0x167
runtime.MCache_Refill(0x89f000, 0x25, 0x800000000)
    /usr/src/go/src/runtime/mcache.c:90 +0xa0

goroutine 37 [running]:
runtime.switchtoM()
    /usr/src/go/src/runtime/asm_amd64.s:198 fp=0xc2080de1f0 sp=0xc2080de1e8
runtime.mallocgc(0x800, 0x3abec0, 0x1, 0x0)
    /usr/src/go/src/runtime/malloc.go:178 +0x849 fp=0xc2080de2a0 sp=0xc2080de1f0
runtime.newarray(0x3abec0, 0x800, 0x0)
    /usr/src/go/src/runtime/malloc.go:365 +0xc1 fp=0xc2080de2d8 sp=0xc2080de2a0
runtime.makeslice(0x39d320, 0x3d1, 0x800, 0x0, 0x0, 0x0)
    /usr/src/go/src/runtime/slice.go:32 +0x15c fp=0xc2080de320 sp=0xc2080de2d8
crypto/tls.(*block).reserve(0xc208082cc0, 0x452)
    /usr/src/go/src/crypto/tls/conn.go:438 +0x74 fp=0xc2080de3a0 sp=0xc2080de320
crypto/tls.(*block).readFromUntil(0xc208082cc0, 0x8b2988, 0xc20803c0a0, 0x452, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/conn.go:452 +0x73 fp=0xc2080de408 sp=0xc2080de3a0
crypto/tls.(*Conn).readRecord(0xc20807a2c0, 0x16, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/conn.go:586 +0xc60 fp=0xc2080de950 sp=0xc2080de408
crypto/tls.(*Conn).readHandshake(0xc20807a2c0, 0x0, 0x0, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/conn.go:779 +0xe7 fp=0xc2080deb10 sp=0xc2080de950
crypto/tls.(*clientHandshakeState).doFullHandshake(0xc208070480, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/handshake_client.go:227 +0x68 fp=0xc2080df0a8 sp=0xc2080deb10
crypto/tls.(*Conn).clientHandshake(0xc20807a2c0, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/handshake_client.go:197 +0x1981 fp=0xc2080df528 sp=0xc2080df0a8
crypto/tls.(*Conn).Handshake(0xc20807a2c0, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/conn.go:977 +0xf1 fp=0xc2080df568 sp=0xc2080df528
crypto/tls.DialWithDialer(0xc208044800, 0x53cdf0, 0x3, 0xc2080587d0, 0x50, 0xc2080d07e0, 0x50dce0, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/tls.go:141 +0x495 fp=0xc2080df678 sp=0xc2080df568
crypto/tls.Dial(0x53cdf0, 0x3, 0xc2080587d0, 0x50, 0xc2080d06c0, 0x0, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/tls.go:165 +0x82 fp=0xc2080df6c8 sp=0xc2080df678
github.com/AdRoll/hologram/transport/remote.NewClient(0xc2080587d0, 0x50, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/AdRoll/hologram/transport/remote/client.go:44 +0x260 fp=0xc2080df780 sp=0xc2080df6c8
github.com/AdRoll/hologram/agent.(*client).requestCredentials(0xc20801ec80, 0xc208044580, 0xc208087760, 0x17, 0x0, 0x0)
    /go/src/github.com/AdRoll/hologram/agent/client.go:169 +0x62 fp=0xc2080df980 sp=0xc2080df780
github.com/AdRoll/hologram/agent.(*client).AssumeRole(0xc20801ec80, 0xc208087760, 0x17, 0x0, 0x0)
    /go/src/github.com/AdRoll/hologram/agent/client.go:157 +0x112 fp=0xc2080df9d0 sp=0xc2080df980
github.com/AdRoll/hologram/agent.(*credentialsExpirationManager).maybeRefreshCredentials(0xc208044480, 0x0, 0x0)
    /go/src/github.com/AdRoll/hologram/agent/credentials_expiration_manager.go:63 +0x1e6 fp=0xc2080dfa68 sp=0xc2080df9d0
github.com/AdRoll/hologram/agent.(*credentialsExpirationManager).GetCredentials(0xc208044480, 0xc200000000, 0x0, 0x0)
    /go/src/github.com/AdRoll/hologram/agent/credentials_expiration_manager.go:48 +0x120 fp=0xc2080dfab8 sp=0xc2080dfa68
github.com/AdRoll/hologram/agent.(*metadataService).getCredentials(0xc20801ec60, 0x8b2a80, 0xc208064500, 0xc20803ad00)
    /go/src/github.com/AdRoll/hologram/agent/metadata_service.go:128 +0x53 fp=0xc2080dfc10 sp=0xc2080dfab8
github.com/AdRoll/hologram/agent.*metadataService.(github.com/AdRoll/hologram/agent.getCredentials)·fm(0x8b2a80, 0xc208064500, 0xc20803ad00)
    /go/src/github.com/AdRoll/hologram/agent/metadata_service.go:62 +0x45 fp=0xc2080dfc38 sp=0xc2080dfc10
net/http.HandlerFunc.ServeHTTP(0xc20800aca0, 0x8b2a80, 0xc208064500, 0xc20803ad00)
    /usr/src/go/src/net/http/server.go:1265 +0x41 fp=0xc2080dfc58 sp=0xc2080dfc38
net/http.(*ServeMux).ServeHTTP(0xc20800d920, 0x8b2a80, 0xc208064500, 0xc20803ad00)
    /usr/src/go/src/net/http/server.go:1541 +0x17d fp=0xc2080dfcb0 sp=0xc2080dfc58
net/http.serverHandler.ServeHTTP(0xc208056300, 0x8b2a80, 0xc208064500, 0xc20803ad00)
    /usr/src/go/src/net/http/server.go:1703 +0x19a fp=0xc2080dfd08 sp=0xc2080dfcb0
net/http.(*conn).serve(0xc208064320)
    /usr/src/go/src/net/http/server.go:1204 +0xb57 fp=0xc2080dffd8 sp=0xc2080dfd08
runtime.goexit()
    /usr/src/go/src/runtime/asm_amd64.s:2232 +0x1 fp=0xc2080dffe0 sp=0xc2080dffd8
created by net/http.(*Server).Serve
    /usr/src/go/src/net/http/server.go:1751 +0x35e

goroutine 1 [select]:
main.main()
    /go/src/github.com/AdRoll/hologram/cmd/hologram-agent/main.go:122 +0x1376

goroutine 5 [syscall]:
os/signal.loop()
    /usr/src/go/src/os/signal/signal_unix.go:21 +0x1f
created by os/signal.init·1
    /usr/src/go/src/os/signal/signal_unix.go:27 +0x35

goroutine 6 [IO wait]:
net.(*pollDesc).Wait(0xc208011250, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208011250, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc2080111f0, 0x0, 0x8b0be0, 0xc208088018)
    /usr/src/go/src/net/fd_unix.go:419 +0x40b
net.(*TCPListener).AcceptTCP(0xc20803c078, 0xc20805edf8, 0x0, 0x0)
    /usr/src/go/src/net/tcpsock_posix.go:234 +0x4e
net.(*TCPListener).Accept(0xc20803c078, 0x0, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/tcpsock_posix.go:244 +0x4c
net/http.(*Server).Serve(0xc208056300, 0x8b2708, 0xc20803c078, 0x0, 0x0)
    /usr/src/go/src/net/http/server.go:1728 +0x92
net/http.Serve(0x8b2708, 0xc20803c078, 0x8b2888, 0xc20800d920, 0x0, 0x0)
    /usr/src/go/src/net/http/server.go:1606 +0xa1
github.com/AdRoll/hologram/agent.(*metadataService).listen(0xc20801ec60)
    /go/src/github.com/AdRoll/hologram/agent/metadata_service.go:67 +0x3be
created by github.com/AdRoll/hologram/agent.(*metadataService).Start
    /go/src/github.com/AdRoll/hologram/agent/metadata_service.go:50 +0x44

goroutine 7 [IO wait]:
net.(*pollDesc).Wait(0xc2080112c0, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc2080112c0, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc208011260, 0x0, 0x8b0be0, 0xc208088748)
    /usr/src/go/src/net/fd_unix.go:419 +0x40b
net.(*UnixListener).AcceptUnix(0xc20801ed40, 0x36aa0, 0x0, 0x0)
    /usr/src/go/src/net/unixsock_posix.go:282 +0x56
net.(*UnixListener).Accept(0xc20801ed40, 0x0, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/unixsock_posix.go:293 +0x4c
github.com/AdRoll/hologram/transport/local.(*server).listen(0xc20801ed60)
    /go/src/github.com/AdRoll/hologram/transport/local/server.go:33 +0x4f
created by github.com/AdRoll/hologram/transport/local.NewServer
    /go/src/github.com/AdRoll/hologram/transport/local/server.go:65 +0x121

goroutine 20 [sleep]:
net.func·019()
    /usr/src/go/src/net/dnsclient_unix.go:240 +0x5a
created by net.loadConfig
    /usr/src/go/src/net/dnsclient_unix.go:269 +0x20c
failed MSpanList_Insert 0x8a3000 0x2f9cbb56eb615 0x0
fatal error: MSpanList_Insert

runtime stack:
runtime.throw(0x7be56b)
    /usr/src/go/src/runtime/panic.go:491 +0xad fp=0x7fff5fbffa50 sp=0x7fff5fbffa20
runtime.MSpanList_Insert(0x7eaa48, 0x8a3000)
    /usr/src/go/src/runtime/mheap.c:692 +0x8f fp=0x7fff5fbffa78 sp=0x7fff5fbffa50
MHeap_FreeSpanLocked(0x7e7640, 0x8a3000, 0x100)
    /usr/src/go/src/runtime/mheap.c:583 +0x163 fp=0x7fff5fbffab8 sp=0x7fff5fbffa78
MHeap_Grow(0x7e7640, 0x8, 0x0)
    /usr/src/go/src/runtime/mheap.c:420 +0x1a8 fp=0x7fff5fbffaf8 sp=0x7fff5fbffab8
MHeap_AllocSpanLocked(0x7e7640, 0x1, 0x0)
    /usr/src/go/src/runtime/mheap.c:298 +0x365 fp=0x7fff5fbffb38 sp=0x7fff5fbffaf8
mheap_alloc(0x7e7640, 0x1, 0x12, 0x0)
    /usr/src/go/src/runtime/mheap.c:190 +0x121 fp=0x7fff5fbffb60 sp=0x7fff5fbffb38
runtime.MHeap_Alloc(0x7e7640, 0x1, 0x10000000012, 0xe8e9)
    /usr/src/go/src/runtime/mheap.c:240 +0x66 fp=0x7fff5fbffb98 sp=0x7fff5fbffb60
MCentral_Grow(0x7ef3b8, 0x0)
    /usr/src/go/src/runtime/mcentral.c:197 +0x8b fp=0x7fff5fbffc00 sp=0x7fff5fbffb98
runtime.MCentral_CacheSpan(0x7ef3b8, 0x0)
    /usr/src/go/src/runtime/mcentral.c:85 +0x167 fp=0x7fff5fbffc38 sp=0x7fff5fbffc00
runtime.MCache_Refill(0x89f000, 0x12, 0x0)
    /usr/src/go/src/runtime/mcache.c:90 +0xa0 fp=0x7fff5fbffc60 sp=0x7fff5fbffc38
runtime.mcacheRefill_m()
    /usr/src/go/src/runtime/malloc.c:368 +0x57 fp=0x7fff5fbffc80 sp=0x7fff5fbffc60
runtime.onM(0x632bf8)
    /usr/src/go/src/runtime/asm_amd64.s:273 +0x9a fp=0x7fff5fbffc88 sp=0x7fff5fbffc80
runtime.mallocgc(0x120, 0x518020, 0x0, 0x0)
    /usr/src/go/src/runtime/malloc.go:178 +0x849 fp=0x7fff5fbffd38 sp=0x7fff5fbffc88
runtime.newobject(0x518020, 0x89f000)
    /usr/src/go/src/runtime/malloc.go:353 +0x49 fp=0x7fff5fbffd60 sp=0x7fff5fbffd38
runtime.newG(0x283ba)
    /usr/src/go/src/runtime/proc.go:233 +0x2a fp=0x7fff5fbffd78 sp=0x7fff5fbffd60
allocg(0x7d7520)
    /usr/src/go/src/runtime/proc.c:925 +0x1f fp=0x7fff5fbffd88 sp=0x7fff5fbffd78
runtime.malg(0x8000, 0x7d76e0)
    /usr/src/go/src/runtime/proc.c:2106 +0x1f fp=0x7fff5fbffdb8 sp=0x7fff5fbffd88
runtime.mpreinit(0x7d7f20)
    /usr/src/go/src/runtime/os_darwin.c:137 +0x27 fp=0x7fff5fbffdd0 sp=0x7fff5fbffdb8
mcommoninit(0x7d7f20)
    /usr/src/go/src/runtime/proc.c:201 +0xc9 fp=0x7fff5fbffdf8 sp=0x7fff5fbffdd0
runtime.schedinit()
    /usr/src/go/src/runtime/proc.c:138 +0x55 fp=0x7fff5fbffe20 sp=0x7fff5fbffdf8
runtime.rt0_go(0x7fff5fbffe50, 0x1, 0x7fff5fbffe50, 0x0, 0x1, 0x7fff5fbffee0, 0x0, 0x7fff5fbffefe, 0x7fff5fbfff21, 0x7fff5fbfff46, ...)
    /usr/src/go/src/runtime/asm_amd64.s:95 +0x116 fp=0x7fff5fbffe28 sp=0x7fff5fbffe20
copumpkin commented 7 years ago

Is nobody else seeing this behavior?

copumpkin commented 7 years ago

Ah, so I don't really know how Docker is supposed to work, but something is wonky with the current setup.

I added a call to go version to start.sh and sure enough, it says "go1.4.2".

Furthermore, when I tinker with the Dockerfile, the hologram.sh script doesn't seem to notice, so the script doesn't appear to be using the Dockerfile in the repository.

When I ran docker rmi adroll/hologram_env and it redownloaded the image, all was well and I was on 1.7.1.

walterking commented 7 years ago

does that solve the crash? I've been running in sierra with no issues, but I dont use the docker build