search

cockroachdb / cockroach

CockroachDB — the cloud native, distributed SQL database designed for high availability, effortless scale, and control over data placement.
https://www.cockroachlabs.com
Other
29.85k stars 3.77k forks source link

teamcity: failed tests on master: acceptance (proposer evaluated kv)/TestPut #12740

Closed cockroach-teamcity closed 7 years ago

cockroach-teamcity commented 7 years ago

The following tests appear to have failed:

#110121:

--- FAIL: acceptance (proposer evaluated kv)/TestPut (368.075s)
Test ended in panic.
panic: test timed out after 10m0s

goroutine 11994 [running]:
panic(0x167ede0, 0xc420be7000)
    /usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
    /usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
    /usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive, 6 minutes]:
testing.(*T).Run(0xc420094180, 0x1863f64, 0x7, 0x1a060b8, 0xc4204d3c01)
    /usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc420094180)
    /usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc420094180, 0xc420368da8)
    /usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x1a09450, 0x2361540, 0x25, 0x25, 0xc4204d3e00)
    /usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc420368ef8, 0x1a05fd0)
    /usr/local/go/src/testing/testing.go:743 +0x85
github.com/cockroachdb/cockroach/pkg/acceptance.RunTests(0xc420368ef8)
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/util.go:150 +0x48
github.com/cockroachdb/cockroach/pkg/acceptance.TestMain(0xc420368ef8)
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/main_test.go:38 +0x45
main.main()
    github.com/cockroachdb/cockroach/pkg/acceptance/_test/_testmain.go:124 +0xc6

goroutine 17 [syscall, 10 minutes, locked to thread]:
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 35 [syscall]:
syscall.Syscall(0x4a, 0x4, 0x0, 0x0, 0xc42002c6d0, 0xbd9bb5, 0xc4205161e8)
    /usr/local/go/src/syscall/asm_linux_amd64.s:18 +0x5
syscall.Fsync(0x4, 0x0, 0x0)
    /usr/local/go/src/syscall/zsyscall_linux_amd64.go:492 +0x4a
os.(*File).Sync(0xc4205161e8, 0x0, 0x0)
    /usr/local/go/src/os/file_posix.go:121 +0x3e
github.com/cockroachdb/cockroach/pkg/util/log.(*syncBuffer).Sync(0xc420bfbb30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:895 +0x2f
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).flushAll(0x28499c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1036 +0x8f
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).lockAndFlushAll(0x28499c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1024 +0x44
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).flushDaemon(0x28499c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1017 +0xa8
created by github.com/cockroachdb/cockroach/pkg/util/log.init.1
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:581 +0x95

goroutine 36 [syscall, 10 minutes]:
os/signal.signal_recv(0x0)
    /usr/local/go/src/runtime/sigqueue.go:116 +0x157
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
    /usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 46 [select, 10 minutes, locked to thread]:
runtime.gopark(0x1a09d08, 0x0, 0x186325e, 0x6, 0x18, 0x2)
    /usr/local/go/src/runtime/proc.go:259 +0x13a
runtime.selectgoImpl(0xc42049e730, 0x0, 0x18)
    /usr/local/go/src/runtime/select.go:423 +0x11d9
runtime.selectgo(0xc42049e730)
    /usr/local/go/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
    /usr/local/go/src/runtime/signal1_unix.go:304 +0x2f3
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 6 [chan receive, 10 minutes]:
github.com/cockroachdb/cockroach/pkg/acceptance.RunTests.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/util.go:141 +0xb6
created by github.com/cockroachdb/cockroach/pkg/acceptance.RunTests
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/util.go:149 +0x3a

goroutine 3391 [select, 5 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc420dfaff0, 0x1a09280, 0xc420e2f4a8, 0x2387600)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:239 +0x299
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x0, 0x0, 0x2380300, 0x2885410, 0x0, 0x0, 0x0, 0x0, 0x2372980, 0xc420b4a460, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:61 +0xaa
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7fc86dfcb650, 0xc420014550, 0x1881962, 0x21, 0x1843aa0, 0xc420a87650, 0x17cbaa0, 0xc4206d2840, 0xc42037c240, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:208 +0x8a1
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7fc86dfcb650, 0xc420014550, 0x1881962, 0x21, 0x1843aa0, 0xc420a87650, 0x17cbaa0, 0xc4206d2840, 0xc42037c240, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:118 +0x19c
github.com/cockroachdb/cockroach/pkg/roachpb.(*externalClient).Batch(0xc4205160d8, 0x7fc86dfcb650, 0xc420014550, 0xc420a87650, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1566 +0xd2
github.com/cockroachdb/cockroach/pkg/internal/client.sender.Send(0x236b780, 0xc4205160d8, 0x7fc86dfcb650, 0xc420014550, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/rpc_sender.go:44 +0xbf
github.com/cockroachdb/cockroach/pkg/internal/client.(*sender).Send(0xc42086ba10, 0x7fc86dfcb650, 0xc420014550, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    <autogenerated>:4 +0xb0
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc420854d20, 0x7fc86dfcb650, 0xc420014550, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:513 +0xbf
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run.func1(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:447 +0x6e
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0xc420b4a520, 0xc420e22600, 0x0, 0x2387d80)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:418 +0x168
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc420854d20, 0x7fc86dfcb650, 0xc420014550, 0xc420e22600, 0xc420b4a500, 0xc420b4a500)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:449 +0xac
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).CheckConsistency(0xc420854d20, 0x7fc86dfcb650, 0xc420014550, 0x1798d40, 0xc420b4a4e0, 0x1798d40, 0xc420b4a500, 0xc400000000, 0x186679b, 0x9)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:398 +0xb7
github.com/cockroachdb/cockroach/pkg/acceptance/cluster.Consistent(0x7fc86dfcb650, 0xc420014550, 0x238a740, 0xc420324500, 0x0, 0x2, 0x2)
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/cluster/cluster.go:80 +0x12c
github.com/cockroachdb/cockroach/pkg/acceptance.testPutInner(0x7fc86dfcb650, 0xc420014550, 0xc420169680, 0x238a740, 0xc420324500, 0xc420812300, 0x9, 0xc420903680, 0x1, 0x1, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/put_test.go:85 +0x68c
github.com/cockroachdb/cockroach/pkg/acceptance.runTestOnConfigs.func1(0x7fc86dfcb650, 0xc420014550, 0xc420169680, 0xc420e2fea8, 0x1a061b8)
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/util.go:338 +0x188
github.com/cockroachdb/cockroach/pkg/acceptance.runTestOnConfigs(0xc420169680, 0x1a061b8)
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/util.go:339 +0x132
github.com/cockroachdb/cockroach/pkg/acceptance.TestPut(0xc420169680)
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/put_test.go:39 +0xad
testing.tRunner(0xc420169680, 0x1a060b8)
    /usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
    /usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 133 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e002038, 0x72, 0x5)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4203a15d0, 0x72, 0xc4200469d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4203a15d0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4203a1570, 0xc42048e000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420034150, 0xc42048e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420142300, 0xc42048e000, 0x1000, 0x1000, 0x30, 0xc420046b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420424180)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc420424180, 0x1, 0x0, 0x1, 0x0, 0xc4203c2660, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420142300)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2644 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86b05e340, 0x72, 0x24)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420328990, 0x72, 0xc42097f818, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420328990, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420328930, 0xc420887000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420516208, 0xc420887000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420324600, 0xc420887000, 0x1000, 0x1000, 0x7622a8, 0xc420380d48, 0x7d)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420a11ec0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).ReadSlice(0xc420a11ec0, 0x10a, 0x2, 0x0, 0x199, 0x186, 0x186)
    /usr/local/go/src/bufio/bufio.go:330 +0xb5
net/http/internal.readChunkLine(0xc420a11ec0, 0x0, 0xc420960598, 0x8, 0x27a, 0xa)
    /usr/local/go/src/net/http/internal/chunked.go:110 +0x34
net/http/internal.(*chunkedReader).beginChunk(0xc420b62060)
    /usr/local/go/src/net/http/internal/chunked.go:47 +0x32
net/http/internal.(*chunkedReader).Read(0xc420b62060, 0xc4208b4001, 0x5ff, 0x5ff, 0x6411c0, 0xc42097fb70, 0xc42097fb78)
    /usr/local/go/src/net/http/internal/chunked.go:77 +0x7b
net/http.(*body).readLocked(0xc42039a080, 0xc4208b4001, 0x5ff, 0x5ff, 0x2, 0x2, 0x0)
    /usr/local/go/src/net/http/transfer.go:651 +0x61
net/http.(*body).Read(0xc42039a080, 0xc4208b4001, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/http/transfer.go:643 +0xf6
net/http.(*bodyEOFSignal).Read(0xc42039a0c0, 0xc4208b4001, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/http/transport.go:1974 +0x116
encoding/json.(*Decoder).refill(0xc420380d00, 0xa, 0x9)
    /usr/local/go/src/encoding/json/stream.go:152 +0xfa
encoding/json.(*Decoder).readValue(0xc420380d00, 0x0, 0x0, 0x80)
    /usr/local/go/src/encoding/json/stream.go:128 +0x198
encoding/json.(*Decoder).Decode(0xc420380d00, 0x1642500, 0xc420272480, 0x0, 0x0)
    /usr/local/go/src/encoding/json/stream.go:57 +0x8e
github.com/cockroachdb/cockroach/vendor/github.com/docker/docker/client.(*Client).Events.func1(0xc420b98120, 0xc420328700, 0x0, 0x0, 0x0, 0x0, 0xc420908030, 0x7fc86e0023f8, 0xc420762a40, 0xc420b980c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/docker/docker/client/events.go:50 +0x2a7
created by github.com/cockroachdb/cockroach/vendor/github.com/docker/docker/client.(*Client).Events
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/docker/docker/client/events.go:63 +0xe2

goroutine 142 [chan receive, 9 minutes]:
database/sql.(*DB).connectionOpener(0xc420185970)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 67 [chan receive, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc42043b340)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 84 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fc400, 0xc420096480, 0xc4201830a0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc42021d350)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1890 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420804580, 0xc420819100, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420804580)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420804580)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 134 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc420142300)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 83 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc42009b760, 0xc420425b00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42009b760)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc42009b760)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 48 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e002278, 0x72, 0x7)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42014e450, 0x72, 0xc4202619d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42014e450, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42014e3f0, 0xc420178000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420136060, 0xc420178000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420324200, 0xc420178000, 0x1000, 0x1000, 0x30, 0xc420261b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4203c2600)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4203c2600, 0x1, 0x0, 0x1, 0x0, 0xc42030f2c0, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420324200)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 49 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc420324200)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 23 [chan receive, 9 minutes]:
database/sql.(*DB).connectionOpener(0xc42032c160)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 102 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420426400, 0xc420485680, 0xc42023b560, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc42021ede0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2338 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc420324700)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 1872 [chan receive, 7 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc42054c7e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 16 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e0020f8, 0x72, 0x9)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4203a2c30, 0x72, 0xc42025e9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4203a2c30, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4203a2bd0, 0xc4200f0000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4201223d8, 0xc4200f0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4204f8600, 0xc4200f0000, 0x1000, 0x1000, 0x9626d0, 0xc42025eb58, 0x5f03ad)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4203bf1a0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4203bf1a0, 0x1, 0xc42025ebbd, 0x1, 0x0, 0xc4204bc180, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4204f8600)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 66 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc4204f8600)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 85 [chan receive, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420131040)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 101 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420159760, 0xc420017000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420159760)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420159760)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 293 [chan receive, 9 minutes]:
database/sql.(*DB).connectionOpener(0xc4207ec420)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 167 [chan receive, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4204bca00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 88 [chan receive, 9 minutes]:
database/sql.(*DB).connectionOpener(0xc4200aeb00)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 114 [chan receive, 9 minutes]:
database/sql.(*DB).connectionOpener(0xc420356160)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 267 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fc680, 0xc4204367e0, 0xc42011f210, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4205f74d0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 151 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc4204e0100)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 141 [chan receive, 9 minutes]:
database/sql.(*DB).connectionOpener(0xc4201853f0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 158 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420426780, 0xc420156a20, 0xc42012f530, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc42045b890)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 150 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e002338, 0x72, 0xe)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420602300, 0x72, 0xc4202639d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420602300, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4206022a0, 0xc420112000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420516040, 0xc420112000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4204e0100, 0xc420112000, 0x1000, 0x1000, 0x9626d0, 0xc420263b58, 0x5f03ad)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4205fc720)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4205fc720, 0x1, 0xc420263bbd, 0x1, 0x0, 0xc4205fca80, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4204e0100)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2305 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86e0067c8, 0x72, 0x32)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420482ed0, 0x72, 0xc4202ac9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420482ed0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420482e70, 0xc420abb000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420034230, 0xc420abb000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420324700, 0xc420abb000, 0x1000, 0x1000, 0x30, 0xc4202acb58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420682780)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc420682780, 0x1, 0x0, 0x1, 0x0, 0xc420a2dc20, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420324700)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2514 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc42074f520)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1651 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420426080, 0xc420484c60, 0xc420551a10, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420245e90)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 157 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420383600, 0xc42030e000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420383600)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420383600)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 1650 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc4204ba160, 0xc4205a5d00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4204ba160)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4204ba160)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 2495 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420426280, 0xc4201fb440, 0xc420514960, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4201f59b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1207 [IO wait, 7 minutes]:
net.runtime_pollWait(0x7fc86dfd0c48, 0x72, 0x21)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4203a9090, 0x72, 0xc4205ce9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4203a9090, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4203a9030, 0xc42076c000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420034190, 0xc42076c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4204e0600, 0xc42076c000, 0x1000, 0x1000, 0x30, 0xc4205ceb58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4204c4b40)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4204c4b40, 0x1, 0x0, 0x1, 0x0, 0xc4205af920, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4204e0600)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 170 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fcc00, 0xc420436b40, 0xc4203925b0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420219f20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 266 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420407a20, 0xc420745200, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420407a20)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420407a20)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 138 [chan receive, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420549260)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 169 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc42042d8c0, 0xc420364c00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42042d8c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc42042d8c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 319 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e001d38, 0x72, 0x14)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420235f70, 0x72, 0xc42058a9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420235f70, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420235f10, 0xc4207f9000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a140, 0xc4207f9000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420508e00, 0xc4207f9000, 0x1000, 0x1000, 0x30, 0xc42058ab58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42027f800)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42027f800, 0x1, 0x0, 0x1, 0x0, 0xc42078fd40, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420508e00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 242 [chan receive, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4205f87a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 175 [chan receive, 9 minutes]:
database/sql.(*DB).connectionOpener(0xc420357550)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3580 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fc380, 0xc4208fa900, 0xc420734690, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420c73380)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 304 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc420508400)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 303 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e001978, 0x72, 0x8)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200ea990, 0x72, 0xc4203459d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200ea990, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200ea930, 0xc4204c2000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a080, 0xc4204c2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420508400, 0xc4204c2000, 0x1000, 0x1000, 0x30, 0xc420345b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4205aefc0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4205aefc0, 0x1, 0x0, 0x1, 0x0, 0xc42027f080, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420508400)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 1695 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86dfd0288, 0x72, 0x2d)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4203a85a0, 0x72, 0xc4206a19d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4203a85a0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4203a8540, 0xc4202b7000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420136090, 0xc4202b7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4204f8100, 0xc4202b7000, 0x1000, 0x1000, 0x30, 0xc4206a1b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4206fe5a0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4206fe5a0, 0x1, 0x0, 0x1, 0x0, 0xc4207d4480, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4204f8100)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2132 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420aca000, 0xc4207dbe00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420aca000)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420aca000)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3514 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420426d80, 0xc4207366c0, 0xc4208be4b0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420afe840)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1463 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420272f00, 0xc4203dc5a0, 0xc420515650, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420668780)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1627 [chan receive, 7 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4206583a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 320 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc420508e00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 245 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fda80, 0xc4200966c0, 0xc42023ad20, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420162600)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 107 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e0018b8, 0x72, 0xa)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200eb100, 0x72, 0xc4203699d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200eb100, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200eb0a0, 0xc42024f000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a0a8, 0xc42024f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4204e0c00, 0xc42024f000, 0x1000, 0x1000, 0x30, 0xc420369b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4207db500)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4207db500, 0x1, 0x0, 0x1, 0x0, 0xc4205a41e0, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4204e0c00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 108 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc4204e0c00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 244 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420406dc0, 0xc420591e00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420406dc0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420406dc0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 1865 [chan receive, 7 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4202f2f40)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2219 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc420324b00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 249 [chan receive, 9 minutes]:
database/sql.(*DB).connectionOpener(0xc4202c2840)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 2238 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4202805a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 264 [chan receive, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4204bc760)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 290 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e0017f8, 0x72, 0xf)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4204c12c0, 0x72, 0xc4200439d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4204c12c0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4204c1260, 0xc42065b000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4201361a0, 0xc42065b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420508d00, 0xc42065b000, 0x1000, 0x1000, 0x30, 0xc420043b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42020d860)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42020d860, 0x1, 0x0, 0x1, 0x0, 0xc4205ae780, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420508d00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 291 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc420508d00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 3461 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc42037d8c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1015 +0x59d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:235 +0xea6

goroutine 1894 [chan receive, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc42080a4d0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3221 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc420932210)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 907 [IO wait, 8 minutes]:
net.runtime_pollWait(0x7fc86dfd0e88, 0x72, 0x1e)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420289800, 0x72, 0xc4207dd9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420289800, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4202897a0, 0xc42074c000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420136088, 0xc42074c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420508600, 0xc42074c000, 0x1000, 0x1000, 0x30, 0xc4207ddb58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420372a20)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc420372a20, 0x1, 0x0, 0x1, 0x0, 0xc4203fe180, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420508600)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 420 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e001738, 0x72, 0x16)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420771870, 0x72, 0xc4203479d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420771870, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420771810, 0xc4207f8000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4201360d0, 0xc4207f8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420142400, 0xc4207f8000, 0x1000, 0x1000, 0x30, 0xc420347b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4205ae660)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4205ae660, 0x1, 0x0, 0x1, 0x0, 0xc42078f260, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420142400)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 421 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc420142400)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 1410 [IO wait, 7 minutes]:
net.runtime_pollWait(0x7fc86dfd0888, 0x72, 0x26)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4203a03e0, 0x72, 0xc4207a69d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4203a03e0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4203a0380, 0xc4206dd000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420136078, 0xc4206dd000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4204e0300, 0xc4206dd000, 0x1000, 0x1000, 0x9626d0, 0xc4207a6b58, 0x5f03ad)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4200f8660)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4200f8660, 0x1, 0xc4207a6bbd, 0x1, 0x0, 0xc4200f9e60, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4204e0300)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 478 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e001678, 0x72, 0x17)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420322a00, 0x72, 0xc4200409d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420322a00, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4203229a0, 0xc4200da000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4201361a8, 0xc4200da000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420142900, 0xc4200da000, 0x1000, 0x1000, 0x30, 0xc420040b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42078fbc0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42078fbc0, 0x1, 0x0, 0x1, 0x0, 0xc420552a20, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420142900)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 479 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc420142900)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 1411 [select, 7 minutes]:
net/http.(*persistConn).writeLoop(0xc4204e0300)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 1471 [IO wait, 7 minutes]:
net.runtime_pollWait(0x7fc86dfd0408, 0x72, 0x31)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420328370, 0x72, 0xc4205f19d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420328370, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420328310, 0xc4202a9000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420034070, 0xc4202a9000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420324000, 0xc4202a9000, 0x1000, 0x1000, 0x9626d0, 0xc4205f1b58, 0x5f03ad)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4203005a0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4203005a0, 0x1, 0xc4205f1bbd, 0x1, 0x0, 0xc42028cf00, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420324000)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 1824 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc42000c980, 0xc42052e7e0, 0xc420213bd0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420668ed0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 529 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e0015b8, 0x72, 0x18)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420394bc0, 0x72, 0xc4200419d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420394bc0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420394b60, 0xc420530000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a1a8, 0xc420530000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4204e0900, 0xc420530000, 0x1000, 0x1000, 0x30, 0xc420041b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42035df80)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42035df80, 0x1, 0x0, 0x1, 0x0, 0xc4205fca80, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4204e0900)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 562 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e0014f8, 0x72, 0x19)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42023f090, 0x72, 0xc42025d9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42023f090, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42023f030, 0xc420766000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a0b8, 0xc420766000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420508500, 0xc420766000, 0x1000, 0x1000, 0x30, 0xc42025db58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42030f020)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42030f020, 0x1, 0x0, 0x1, 0x0, 0xc4207dbe60, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420508500)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 3300 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc420143b00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 546 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc4204e0900)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 2249 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420667ce0, 0xc420a2c300, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420667ce0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420667ce0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 1472 [select, 7 minutes]:
net/http.(*persistConn).writeLoop(0xc420324000)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 843 [select, 8 minutes]:
net/http.(*persistConn).writeLoop(0xc4204e0e00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 3057 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420946300)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1141 [IO wait, 8 minutes]:
net.runtime_pollWait(0x7fc86dfd0d08, 0x72, 0x20)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200eb5d0, 0x72, 0xc4205889d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200eb5d0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200eb570, 0xc420764000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a100, 0xc420764000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420142a00, 0xc420764000, 0x1000, 0x1000, 0x30, 0xc420588b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4205a4600)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4205a4600, 0x1, 0x0, 0x1, 0x0, 0xc420553140, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420142a00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 606 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc420142d00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 605 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86e001438, 0x72, 0x1a)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420323f70, 0x72, 0xc4203429d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420323f70, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420323f10, 0xc4205ac000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4200341a8, 0xc4205ac000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420142d00, 0xc4205ac000, 0x1000, 0x1000, 0x30, 0xc420342b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4205527e0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4205527e0, 0x1, 0x0, 0x1, 0x0, 0xc420519140, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420142d00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 563 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc420508500)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 1375 [chan receive, 7 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4206596a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 990 [select, 8 minutes]:
net/http.(*persistConn).writeLoop(0xc420142600)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 811 [IO wait, 8 minutes]:
net.runtime_pollWait(0x7fc86dfd1008, 0x72, 0x1b)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420010140, 0x72, 0xc4207e19d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420010140, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200100e0, 0xc420306000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a0a0, 0xc420306000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4204e0400, 0xc420306000, 0x1000, 0x1000, 0x30, 0xc4207e1b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4204c5a40)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4204c5a40, 0x1, 0x0, 0x1, 0x0, 0xc4205a28a0, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4204e0400)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 1208 [select, 7 minutes]:
net/http.(*persistConn).writeLoop(0xc4204e0600)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 1891 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc42000d800, 0xc420156360, 0xc4206a2f40, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4202c6870)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 842 [IO wait, 8 minutes]:
net.runtime_pollWait(0x7fc86dfd10c8, 0x72, 0x1c)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4204c0990, 0x72, 0xc4207e09d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4204c0990, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4204c0930, 0xc4207d2000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a160, 0xc4207d2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4204e0e00, 0xc4207d2000, 0x1000, 0x1000, 0x30, 0xc4207e0b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4200f8ea0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4200f8ea0, 0x1, 0x0, 0x1, 0x0, 0xc42039eba0, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4204e0e00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2480 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc420af2d10)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 1302 [IO wait, 7 minutes]:
net.runtime_pollWait(0x7fc86dfd0b88, 0x72, 0x22)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42060a920, 0x72, 0xc4207e29d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42060a920, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42060a8c0, 0xc42039e000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a0b0, 0xc42039e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420142700, 0xc42039e000, 0x1000, 0x1000, 0x30, 0xc4207e2b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4202c52c0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4202c52c0, 0x1, 0x0, 0x1, 0x0, 0xc420364780, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420142700)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 812 [select, 8 minutes]:
net/http.(*persistConn).writeLoop(0xc4204e0400)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 768 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc420142200)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 767 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7fc86dfd0f48, 0x72, 0x1d)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42014e0d0, 0x72, 0xc4203669d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42014e0d0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42014e070, 0xc42033f000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4201360e8, 0xc42033f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420142200, 0xc42033f000, 0x1000, 0x1000, 0x30, 0xc420366b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4205a5e60)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4205a5e60, 0x1, 0x0, 0x1, 0x0, 0xc4205190e0, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420142200)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 908 [select, 8 minutes]:
net/http.(*persistConn).writeLoop(0xc420508600)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 1303 [select, 7 minutes]:
net/http.(*persistConn).writeLoop(0xc420142700)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 2960 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420272200, 0xc42052e900, 0xc420b78ee0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4208b7470)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2959 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420aca2c0, 0xc4207aca00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420aca2c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420aca2c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 1618 [IO wait, 7 minutes]:
net.runtime_pollWait(0x7fc86dfd0708, 0x72, 0x2c)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420394990, 0x72, 0xc4207a49d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420394990, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420394930, 0xc4209eb000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4201223f0, 0xc4209eb000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420508700, 0xc4209eb000, 0x1000, 0x1000, 0x9626d0, 0xc4207a4b58, 0x5f03ad)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42078ed20)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42078ed20, 0x1, 0xc4207a4bbd, 0x1, 0x0, 0xc4207acd80, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420508700)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2494 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc4203822c0, 0xc420a84900, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4203822c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4203822c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 1142 [select, 8 minutes]:
net/http.(*persistConn).writeLoop(0xc420142a00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 989 [IO wait, 8 minutes]:
net.runtime_pollWait(0x7fc86dfd0dc8, 0x72, 0x1f)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42060a5a0, 0x72, 0xc4205d39d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42060a5a0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42060a540, 0xc4205da000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a0c8, 0xc4205da000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420142600, 0xc4205da000, 0x1000, 0x1000, 0x30, 0xc4205d3b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4203fe840)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4203fe840, 0x1, 0x0, 0x1, 0x0, 0xc420642f60, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420142600)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 3535 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc420808000)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1015 +0x59d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:235 +0xea6

goroutine 1619 [select, 7 minutes]:
net/http.(*persistConn).writeLoop(0xc420508700)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 3654 [IO wait]:
net.runtime_pollWait(0x7fc86b05db00, 0x72, 0x55)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420771560, 0x72, 0xc420984840, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420771560, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420771500, 0xc42078c800, 0x400, 0x400, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420516098, 0xc42078c800, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc420aff680, 0x7fc86e0024b0, 0xc420516098, 0x5, 0xc420516098, 0xc420919be0)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc4206d6e00, 0x1a09e17, 0xc4206d6f08, 0x5f0c0e)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc4206d6e00, 0xc420d06000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc42091e720)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc42091e720, 0xc42028aed8, 0x9, 0x9, 0x109e05a, 0xc420c732c8, 0xc4203c3bc0)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x236a8c0, 0xc42091e720, 0xc42028aed8, 0x9, 0x9, 0x9, 0xc420984d38, 0x109f1ce, 0xc420c732c0)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x236a8c0, 0xc42091e720, 0xc42028aed8, 0x9, 0x9, 0x0, 0x0, 0x236b000)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42028aed8, 0x9, 0x9, 0x236a8c0, 0xc42091e720, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42028aea0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:469 +0x8e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4202c6390, 0xc420c73590, 0xc420c73590, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc4205e6120)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:937 +0xb6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:205 +0x8df

goroutine 1823 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc42009adc0, 0xc4206af700, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42009adc0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc42009adc0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 1431 [chan receive, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc42036f3f0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 1447 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc4202e62c0, 0xc4207efc00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4202e62c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4202e62c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 1464 [chan receive, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc42036ef20)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 1432 [chan receive, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc42036f6b0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 1377 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420382c60, 0xc4205aef00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420382c60)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420382c60)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 1426 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fcd00, 0xc420097560, 0xc4204671f0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4201f5bf0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1462 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc4207bb340, 0xc4205aeb00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4207bb340)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4207bb340)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3444 [select, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/acceptance/cluster.(*LocalCluster).monitor.func1(0x1)
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/cluster/localcluster.go:618 +0x4dc
github.com/cockroachdb/cockroach/pkg/acceptance/cluster.(*LocalCluster).monitor(0xc420324500, 0x7fc86dfcb650, 0xc420014550)
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/cluster/localcluster.go:638 +0x84
created by github.com/cockroachdb/cockroach/pkg/acceptance/cluster.(*LocalCluster).Start
    /go/src/github.com/cockroachdb/cockroach/pkg/acceptance/cluster/localcluster.go:662 +0x5a4

goroutine 1336 [IO wait, 7 minutes]:
net.runtime_pollWait(0x7fc86dfd0ac8, 0x72, 0x23)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42060afb0, 0x72, 0xc4202609d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42060afb0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42060af50, 0xc4202bc000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a200, 0xc4202bc000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420143300, 0xc4202bc000, 0x1000, 0x1000, 0x30, 0xc420260b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42027f920)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42027f920, 0x1, 0x0, 0x1, 0x0, 0xc4207ac3c0, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420143300)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 1337 [select, 7 minutes]:
net/http.(*persistConn).writeLoop(0xc420143300)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 2932 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc420143000)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 1448 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420272980, 0xc4201fad80, 0xc42023b5b0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420669410)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2218 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86e006408, 0x72, 0x28)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420a128b0, 0x72, 0xc4207a79d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420a128b0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420a12850, 0xc420a18000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a238, 0xc420a18000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420324b00, 0xc420a18000, 0x1000, 0x1000, 0x30, 0xc4207a7b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420a10ba0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc420a10ba0, 0x1, 0x0, 0x1, 0x0, 0xc420682a80, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420324b00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2957 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4207791a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1449 [chan receive, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc42032cdc0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 1423 [chan receive, 7 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420404200)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2134 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420242140)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1404 [IO wait, 7 minutes]:
net.runtime_pollWait(0x7fc86dfd0948, 0x72, 0x25)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4203a2530, 0x72, 0xc4202ae9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4203a2530, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4203a24d0, 0xc420473000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420034098, 0xc420473000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420142c00, 0xc420473000, 0x1000, 0x1000, 0x30, 0xc4202aeb58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4207d47e0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4207d47e0, 0x1, 0x0, 0x1, 0x0, 0xc4207be0c0, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420142c00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 1405 [select, 7 minutes]:
net/http.(*persistConn).writeLoop(0xc420142c00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 3373 [IO wait, 5 minutes]:
net.runtime_pollWait(0x7fc86b05e1c0, 0x72, 0x4c)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420a123e0, 0x72, 0xc420b169d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420a123e0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420a12380, 0xc420ab6000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4201223e0, 0xc420ab6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000af00, 0xc420ab6000, 0x1000, 0x1000, 0x9626d0, 0xc420b16b58, 0x5f03ad)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420912600)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc420912600, 0x1, 0xc420b16bbd, 0x1, 0x0, 0xc420818a20, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000af00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 1634 [chan receive, 7 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420698cc0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2137 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420426480, 0xc420156240, 0xc42086a020, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420668a20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1611 [IO wait, 7 minutes]:
net.runtime_pollWait(0x7fc86dfd0348, 0x72, 0x27)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4203a05a0, 0x72, 0xc4202b19d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4203a05a0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4203a0540, 0xc4202a4000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a070, 0xc4202a4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420142800, 0xc4202a4000, 0x1000, 0x1000, 0x30, 0xc4202b1b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4207d4b40)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4207d4b40, 0x1, 0x0, 0x1, 0x0, 0xc42078e840, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420142800)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 1612 [select, 7 minutes]:
net/http.(*persistConn).writeLoop(0xc420142800)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 1696 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc4204f8100)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 1654 [chan receive, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc4202c36b0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 1655 [chan receive, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc4202c3810)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 1839 [chan receive, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc4209e88f0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 2129 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420657e80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3218 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc42000c780, 0xc4206f0480, 0xc420918850, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420bfb800)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2254 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc420a9a630)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3203 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420405f20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2133 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc42000cd00, 0xc4205e66c0, 0xc420858110, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc42015fdd0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3579 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420ac06e0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:751 +0x63e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420ac06e0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3495 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4208414c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 1883 [chan receive, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc420824790)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3543 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420acaf20)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:751 +0x63e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420acaf20)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 2931 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86dfd01c8, 0x72, 0x3a)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420483100, 0x72, 0xc420a0c9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420483100, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4204830a0, 0xc420b70000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420122438, 0xc420b70000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420143000, 0xc420b70000, 0x1000, 0x1000, 0x30, 0xc420a0cb58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4200f8de0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4200f8de0, 0x1, 0x0, 0x1, 0x0, 0xc4207eea80, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420143000)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 3299 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86b05e580, 0x72, 0x46)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420922920, 0x72, 0xc4206a09d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420922920, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4209228c0, 0xc4206b6000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a440, 0xc4206b6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420143b00, 0xc4206b6000, 0x1000, 0x1000, 0x30, 0xc4206a0b58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42020cba0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42020cba0, 0x1, 0x0, 0x1, 0x0, 0xc4207426c0, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420143b00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 3201 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420a36160, 0xc420960100, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420a36160)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420a36160)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3517 [IO wait]:
net.runtime_pollWait(0x7fc86b05e4c0, 0x72, 0x4a)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420a86530, 0x72, 0xc42093f840, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420a86530, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420a864d0, 0xc420a32800, 0x400, 0x400, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420516188, 0xc420a32800, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc420afe9f0, 0x7fc86e0024b0, 0xc420516188, 0x5, 0xc420516188, 0xc42066c670)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc420a02a80, 0x1a09e17, 0xc420a02b88, 0x5f0c0e)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc420a02a80, 0xc42087e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc4206ff740)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc4206ff740, 0xc420740518, 0x9, 0x9, 0x109e05a, 0xc420c72638, 0xc420520f00)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x236a8c0, 0xc4206ff740, 0xc420740518, 0x9, 0x9, 0x9, 0xc42093fd38, 0x109f1ce, 0xc420c72630)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x236a8c0, 0xc4206ff740, 0xc420740518, 0x9, 0x9, 0x0, 0x0, 0x236b000)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420740518, 0x9, 0x9, 0x236a8c0, 0xc4206ff740, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4207404e0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:469 +0x8e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420aff590, 0xc420c72750, 0xc420c72750, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc42052ed80)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:937 +0xb6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:205 +0x8df

goroutine 2136 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420ac0420, 0xc42028dc00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420ac0420)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420ac0420)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 2346 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420124400, 0xc420436ea0, 0xc4206e02d0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420286d50)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3518 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc42052ed80)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1015 +0x59d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:235 +0xea6

goroutine 2141 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420595400)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2144 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fd380, 0xc4203dcd80, 0xc4206e1dc0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4202876b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2143 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420ac1080, 0xc4206d2600, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420ac1080)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420ac1080)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3173 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc42000b900)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 2250 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420426580, 0xc420097320, 0xc4205e4120, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420313ec0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2345 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420805ce0, 0xc420301e00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420805ce0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420805ce0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3317 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86b05e100, 0x72, 0x4d)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200ea3e0, 0x72, 0xc4209869d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200ea3e0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200ea380, 0xc42066b000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420516090, 0xc42066b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420458600, 0xc42066b000, 0x1000, 0x1000, 0x9626d0, 0xc420986b58, 0x5f03ad)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420baa600)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc420baa600, 0x1, 0xc420986bbd, 0x1, 0x0, 0xc420b64120, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420458600)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2371 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc42080b970)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 2366 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4205971a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3336 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420845be0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3318 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc420458600)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 2500 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc4209e8e70)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 2465 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc420509100)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 2464 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86e005e08, 0x72, 0x37)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4204c0220, 0x72, 0xc4202aa9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4204c0220, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4204c01c0, 0xc4209ea000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420516110, 0xc4209ea000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420509100, 0xc4209ea000, 0x1000, 0x1000, 0x30, 0xc4202aab58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420743aa0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc420743aa0, 0x1, 0x0, 0x1, 0x0, 0xc4206da780, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420509100)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2499 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc4209e8d10)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3627 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fc500, 0xc4208fa000, 0xc420735540, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420e9a960)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3172 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86e006108, 0x72, 0x41)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42023f330, 0x72, 0xc420a0d9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42023f330, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42023f2d0, 0xc4208e0000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a400, 0xc4208e0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000b900, 0xc4208e0000, 0x1000, 0x1000, 0x30, 0xc420a0db58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42091f140)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42091f140, 0x1, 0x0, 0x1, 0x0, 0xc420742600, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000b900)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2415 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86e006348, 0x72, 0x39)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420174450, 0x72, 0xc4207a19d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420174450, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4201743f0, 0xc42046f000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a060, 0xc42046f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4204f8200, 0xc42046f000, 0x1000, 0x1000, 0x9626d0, 0xc4207a1b58, 0x5f03ad)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4206da5a0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4206da5a0, 0x1, 0xc4207a1bbd, 0x1, 0x0, 0xc4206dbaa0, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4204f8200)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 2416 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc4204f8200)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 2540 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4209feae0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2543 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc42000db80, 0xc42052e120, 0xc42012e8d0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc42021c810)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2542 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc4209f49a0, 0xc420b19d00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4209f49a0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4209f49a0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 2544 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc42080a210)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 2562 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc42080afd0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 2563 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc42080bad0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 2630 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4208eb000)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3513 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420a36c60)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:751 +0x63e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420a36c60)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3374 [select, 5 minutes]:
net/http.(*persistConn).writeLoop(0xc42000af00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 3353 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420124480, 0xc4208fb440, 0xc4208582d0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4202c77a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3352 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420a5e840, 0xc420913500, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420a5e840)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420a5e840)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 2961 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc420af40b0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3337 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc4200fa630)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3048 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc420324d00)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 3047 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fc86e005ec8, 0x72, 0x3b)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420234d10, 0x72, 0xc4205ee9d0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420234d10, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420234cb0, 0xc420bff000, 0x1000, 0x1000, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42034a240, 0xc420bff000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc420324d00, 0xc420bff000, 0x1000, 0x1000, 0x30, 0xc4205eeb58, 0x62763c)
    /usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420760c00)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc420760c00, 0x1, 0x0, 0x1, 0x0, 0xc420743ce0, 0x0)
    /usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc420324d00)
    /usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 3599 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc420736900)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1015 +0x59d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:235 +0xea6

goroutine 3076 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fdd00, 0xc4208fa120, 0xc4206e88c0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420aff110)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3075 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc420407e40, 0xc420961200, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:714 +0x72d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420407e40)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:794 +0x362
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420407e40)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3077 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc420a7abb0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3393 [select, 6 minutes]:
net/http.(*persistConn).readLoop(0xc420324600)
    /usr/local/go/src/net/http/transport.go:1541 +0x9c9
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 3544 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fdd80, 0xc4208fbb00, 0xc42086ac40, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4202c7ef0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3410 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc420324600)
    /usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 3569 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc42052eea0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1015 +0x59d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:235 +0xea6

goroutine 3537 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc420a17b80)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3533 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420a355e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3534 [IO wait]:
net.runtime_pollWait(0x7fc86b05e040, 0x72, 0x4e)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420483480, 0x72, 0xc4207a0840, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420483480, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420483420, 0xc420b72400, 0x400, 0x400, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420b2c138, 0xc420b72400, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc4206685d0, 0x7fc86e0024b0, 0xc420b2c138, 0x5, 0xc420b2c138, 0xc4208584f0)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc420327880, 0x1a09e17, 0xc420327988, 0x5f0c0e)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc420327880, 0xc420a1a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc4206d20c0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc4206d20c0, 0xc4200f21d8, 0x9, 0x9, 0x109e05a, 0xc420c72ae8, 0xc4203c21e0)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x236a8c0, 0xc4206d20c0, 0xc4200f21d8, 0x9, 0x9, 0x9, 0xc4207a0d38, 0x109f1ce, 0xc420c72ae0)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x236a8c0, 0xc4206d20c0, 0xc4200f21d8, 0x9, 0x9, 0x0, 0x0, 0x236b000)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4200f21d8, 0x9, 0x9, 0x236a8c0, 0xc4206d20c0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4200f21a0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:469 +0x8e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420b9cc30, 0xc420c72b40, 0xc420c72b40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc420808000)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:937 +0xb6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:205 +0x8df

goroutine 3478 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc420b7eb00)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3547 [chan receive, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc420af5ad0)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 3548 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc42038e860)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3481 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420bf4280, 0xc4206f10e0, 0xc4202b8f50, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420910f00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3480 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420a5fa20)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:751 +0x63e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420a5fa20)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3460 [IO wait]:
net.runtime_pollWait(0x7fc86b05dd40, 0x72, 0x52)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420394ca0, 0x72, 0xc420987840, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420394ca0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420394c40, 0xc4205a2400, 0x400, 0x400, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420034028, 0xc4205a2400, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc420244180, 0x7fc86e0024b0, 0xc420034028, 0x5, 0xc420034028, 0xc4200dc300)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc420654000, 0x1a09e17, 0xc420654108, 0x6b8a87)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc420654000, 0xc42098c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc420b65920)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420b65920, 0xc4201452e8, 0x9, 0x9, 0x109df59, 0xc420c72c68, 0x0)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x236a8c0, 0xc420b65920, 0xc4201452e8, 0x9, 0x9, 0x9, 0xc420987d38, 0x109f1ce, 0xc420c72c60)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x236a8c0, 0xc420b65920, 0xc4201452e8, 0x9, 0x9, 0x0, 0x0, 0x236b000)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4201452e8, 0x9, 0x9, 0x236a8c0, 0xc420b65920, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4201452b0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:469 +0x8e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420b636e0, 0xc420c72db0, 0xc420c72db0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc42037d8c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:937 +0xb6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:205 +0x8df

goroutine 3568 [IO wait]:
net.runtime_pollWait(0x7fc86b05dc80, 0x72, 0x53)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420328290, 0x72, 0xc420941840, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420328290, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420328230, 0xc4206d9800, 0x400, 0x400, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420136058, 0xc4206d9800, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc4209088a0, 0x7fc86e0024b0, 0xc420136058, 0x5, 0xc420136058, 0xc420735310)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc4207abc00, 0x1a09e17, 0xc4207abd08, 0x6b8a87)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc4207abc00, 0xc420acc000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc420b641e0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420b641e0, 0xc42028a518, 0x9, 0x9, 0x109df59, 0xc420b9d7d8, 0x0)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x236a8c0, 0xc420b641e0, 0xc42028a518, 0x9, 0x9, 0x9, 0xc420941d38, 0x109f1ce, 0xc420b9d7d0)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x236a8c0, 0xc420b641e0, 0xc42028a518, 0x9, 0x9, 0x0, 0x0, 0x236b000)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42028a518, 0x9, 0x9, 0x236a8c0, 0xc420b641e0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42028a4e0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:469 +0x8e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4209093e0, 0xc420c72180, 0xc420c72180, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc42052eea0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:937 +0xb6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:205 +0x8df

goroutine 3574 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4203fc300, 0xc420736120, 0xc4206e89e0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4209119b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3651 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420afa7e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3590 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4209029e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3573 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420805340)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:751 +0x63e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420805340)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3626 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420ac1b80)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:751 +0x63e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420ac1b80)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3598 [IO wait]:
net.runtime_pollWait(0x7fc86b05dbc0, 0x72, 0x54)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420816530, 0x72, 0xc420c67840, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420816530, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4208164d0, 0xc420b89000, 0x400, 0x400, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420136118, 0xc420b89000, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc420c73440, 0x7fc86e0024b0, 0xc420136118, 0x5, 0xc420136118, 0xc420918210)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc4206d7880, 0x1a09e17, 0xc4206d7988, 0x5f0c0e)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc4206d7880, 0xc4209f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc420b99560)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420b99560, 0xc4208602a8, 0x9, 0x9, 0x109e05a, 0xc420b625a8, 0xc420364ae0)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x236a8c0, 0xc420b99560, 0xc4208602a8, 0x9, 0x9, 0x9, 0xc420c67d38, 0x109f1ce, 0xc420b625a0)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x236a8c0, 0xc420b99560, 0xc4208602a8, 0x9, 0x9, 0x0, 0x0, 0x236b000)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4208602a8, 0x9, 0x9, 0x236a8c0, 0xc420b99560, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420860270, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:469 +0x8e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420c762a0, 0xc4206eeae0, 0xc4206eeae0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc420736900)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:937 +0xb6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:205 +0x8df

goroutine 2640 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420272000, 0xc42037c240, 0xc42086aaf0, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x629
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc4202453b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3655 [select, 6 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc4205e6120)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1015 +0x59d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:235 +0xea6

goroutine 3690 [chan receive, 5 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420018090, 0xc420afb8c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 2639 [select, 5 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4208049a0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:751 +0x63e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4208049a0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:506 +0x1de
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 3699 [IO wait]:
net.runtime_pollWait(0x7fc86b05da40, 0x72, 0x56)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420a87790, 0x72, 0xc420c6d840, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420a87790, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420a87730, 0xc42078d800, 0x400, 0x400, 0x0, 0x23737c0, 0xc420014068)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420516138, 0xc42078d800, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc420245500, 0x7fc86e0024b0, 0xc420516138, 0x5, 0xc420516138, 0xc4208133d4)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc420877500, 0x1a09e17, 0xc420877608, 0x5f0c0e)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc420877500, 0xc420e34000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc420300c60)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420300c60, 0xc4207413b8, 0x9, 0x9, 0x109e05a, 0xc420b9dce8, 0xc42027f9e0)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x236a8c0, 0xc420300c60, 0xc4207413b8, 0x9, 0x9, 0x9, 0xc420c6dd38, 0x109f1ce, 0xc420b9dce0)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x236a8c0, 0xc420300c60, 0xc4207413b8, 0x9, 0x9, 0x0, 0x0, 0x236b000)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4207413b8, 0x9, 0x9, 0x236a8c0, 0xc420300c60, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420741380, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:469 +0x8e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc42045b410, 0xc420b62000, 0xc420b62000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc42037d9e0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:937 +0xb6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:205 +0x8df

goroutine 3700 [select, 5 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc42037d9e0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1015 +0x59d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:235 +0xea6

Please assign, take a look and update the issue accordingly.

petermattis commented 7 years ago

This was happening on a lot of my PRs and then it just went away. Concurrently, I was unable to run TestPut locally because of cert errors and then that problem went away without any change on my part. Very strange. I'm going to leave this open for a bit in case it reproduces.

a-robinson commented 7 years ago

Happened again in #12796. I'm also going to consolidate the other two issues that are still open for the same timeout (#12738 and #10367).

petermattis commented 7 years ago

The cert issue with TestPut reappeared when I'm running locally. Once per second the acceptance test reports:

I170109 21:09:39.024322 33 vendor/google.golang.org/grpc/clientconn.go:559  Failed to dial 127.0.0.1:32856: connection error: desc = "transport: remote error: tls: bad certificate"; please retry.

Not sure if this is related.

petermattis commented 7 years ago

It's interesting that every TestPut failure that I looked at is blocked on a call to client.(*DB).CheckConsistency. And those calls are all blocked waiting for a gRPC response. This is the only test which calls CheckConsistency so frequently (once per second).

petermattis commented 7 years ago

Kept run into the cert issue locally. Turns out the clock in my docker VM was a few days behind reality. It reported the date as Jan 7, 2017 while the certs were not created until Jan 9, 2017. Restarting docker fixed the issue and I've now run TestPut a few dozen times locally without failure.

petermattis commented 7 years ago

go test -count 100 -tags acceptance -run TestPut -timeout 24h -v --nodes 3 -l . --logtostderr passed on all 100 runs locally. I think I need to add some more debugging info as this failure is happening very frequently on teamcity.

a-robinson commented 7 years ago

Did you ever get around to adding debugging info? It's still timing out pretty regularly.

petermattis commented 7 years ago

Not yet. Still on my TODO list.