search

cockroachdb / cockroach

CockroachDB — the cloud native, distributed SQL database designed for high availability, effortless scale, and control over data placement.
https://www.cockroachlabs.com
Other
30.1k stars 3.8k forks source link

(unknown): (unknown) failed under stress #10893

Closed cockroach-teamcity closed 7 years ago

cockroach-teamcity commented 7 years ago

SHA: https://github.com/cockroachdb/cockroach/commits/425c005d8b55ab834091079937847a1b552e5364

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=54282

    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:495 +0x43
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4210e2060)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 313 [chan receive, 9 minutes]:
github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4201c24e0, 0xc4211d2100, 0x0, 0xc42050c000, 0x6400000000243200)
    <autogenerated>:9 +0x7f
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc420162260, 0xc4201bb320, 0x233b440, 0xc4201c24e0, 0xc4211d2100, 0xa18ddf, 0xc4207b3f88)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:115 +0x5d
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func8()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:508 +0x13c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4210374d0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 314 [chan receive, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:381 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4210e2100)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 315 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:998 +0x46c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420f6d060)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 316 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1027 +0x394
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420f6d070)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 317 [chan receive, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func1(0xc420bd05f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:373 +0x44
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:374 +0x702

goroutine 256 [select, 2 minutes]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc4211b6990)
    /go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c
created by google.golang.org/grpc/transport.newHTTP2Server
    /go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767

goroutine 287 [select, 9 minutes]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc420be23c0)
    /go/src/google.golang.org/grpc/clientconn.go:720 +0x63e
google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420be23c0)
    /go/src/google.golang.org/grpc/clientconn.go:479 +0x1de
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/google.golang.org/grpc/clientconn.go:480 +0x3c7

goroutine 289 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:80 +0x1ed
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc42110ea00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 319 [IO wait]:
net.runtime_pollWait(0x7f3dd12988c8, 0x72, 0x11)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4210d6920, 0x72, 0xc4206e6828, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4210d6920, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4210d68c0, 0xc421fe0000, 0x2000, 0x2000, 0x0, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4203c20a8, 0xc421fe0000, 0x2000, 0x2000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc420ff2390, 0x7f3dd121b258, 0xc4203c20a8, 0x5, 0xc4203c20a8, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc420638a80, 0x1a20017, 0xc420638b88, 0xc420c59d40)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc420638a80, 0xc420d64000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc420ff0d80)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420ff0d80, 0xc420aa79b8, 0x9, 0x9, 0x620340, 0xc4210e56c0, 0xc4206e6d08)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x2325a80, 0xc420ff0d80, 0xc420aa79b8, 0x9, 0x9, 0x9, 0xc420feb328, 0x232a900, 0xc42685f680)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x2325a80, 0xc420ff0d80, 0xc420aa79b8, 0x9, 0x9, 0xc40298, 0xc4212091d0, 0xc420be2140)
    /usr/local/go/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc420aa79b8, 0x9, 0x9, 0x2325a80, 0xc420ff0d80, 0x0, 0xc400000000, 0x0, 0x124)
    /go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc420aa7980, 0x2330dc0, 0xc42685f650, 0x0, 0x0)
    /go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc420feb230, 0xc42685f650, 0xc42685f650, 0x0, 0x0)
    /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Client).reader(0xc42017a600)
    /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6
created by google.golang.org/grpc/transport.newHTTP2Client
    /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1

goroutine 320 [select]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc42017a600)
    /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d
created by google.golang.org/grpc/transport.newHTTP2Client
    /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78

goroutine 321 [select, 9 minutes]:
google.golang.org/grpc.NewClientStream.func2(0x2343ae0, 0xc42017a600, 0xc4212091d0, 0xc4200a8900)
    /go/src/google.golang.org/grpc/stream.go:197 +0x426
created by google.golang.org/grpc.NewClientStream
    /go/src/google.golang.org/grpc/stream.go:217 +0xb5f

goroutine 338 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc4201a8000, 0x2343360, 0xc420c72a70, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:190 +0x57b
github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x180c500, 0xc4201a8000, 0x233f8e0, 0xc420b96cf0, 0x9, 0x9)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:209 +0xbb
google.golang.org/grpc.(*Server).processStreamingRPC(0xc4202c1a40, 0x2343b40, 0xc4211b6990, 0xc4212092c0, 0xc42029d290, 0x2310220, 0x0, 0x0, 0x0)
    /go/src/google.golang.org/grpc/server.go:689 +0x702
google.golang.org/grpc.(*Server).handleStream(0xc4202c1a40, 0x2343b40, 0xc4211b6990, 0xc4212092c0, 0x0)
    /go/src/google.golang.org/grpc/server.go:773 +0xc36
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420670280, 0xc4202c1a40, 0x2343b40, 0xc4211b6990, 0xc4212092c0)
    /go/src/google.golang.org/grpc/server.go:422 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
    /go/src/google.golang.org/grpc/server.go:423 +0xa3

goroutine 354 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420fe1340, 0xc420d3bad0, 0x5, 0x5, 0x0, 0x0, 0x0)
    /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc4212091d0, 0xc420d3bad0, 0x5, 0x5, 0x5ccc4f, 0xc420b29c58, 0x756ea12c7adc00)
    /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x232a7c0, 0xc4212091d0, 0xc420d3bad0, 0x5, 0x5, 0x5, 0x6fdb99, 0xc420165100, 0xc420c59d40)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x232a7c0, 0xc4212091d0, 0xc420d3bad0, 0x5, 0x5, 0xc420165100, 0xc420192360, 0xc420b29c50)
    /usr/local/go/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc420d3bac0, 0x7fffffff, 0xc400000001, 0x28367a0, 0x0, 0x0, 0x0, 0x0)
    /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc420d3bac0, 0x233c140, 0x2835730, 0xc4212091d0, 0x0, 0x0, 0x17a18c0, 0xc428562870, 0x7fffffff, 0xc420034c00, ...)
    /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*clientStream).RecvMsg(0xc4200a8900, 0x17a18c0, 0xc428562870, 0x0, 0x0)
    /go/src/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc420fe9b40, 0x7f3dd1216be8, 0xc4210e5140, 0xc42027e300)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:192 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x2344020, 0xc420fe9b40, 0xc4202b6b00, 0x7f3dd1216be8, 0xc4210e5140, 0xc42027e300, 0x5f58b3, 0xc420d9df48)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:297 +0x35
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:305 +0xd7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4203b8eb0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 355 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420fe1480, 0xc42110f2f0, 0x5, 0x5, 0x0, 0x0, 0x0)
    /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc4212092c0, 0xc42110f2f0, 0x5, 0x5, 0x16ea6a0, 0xc4201a8030, 0x40)
    /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x232a7c0, 0xc4212092c0, 0xc42110f2f0, 0x5, 0x5, 0x5, 0xc420fe14c0, 0x0, 0xc420fe1500)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x232a7c0, 0xc4212092c0, 0xc42110f2f0, 0x5, 0x5, 0xc400000001, 0x0, 0x5da0b9)
    /usr/local/go/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc42110f2e0, 0x7fffffff, 0x6094ab, 0x38, 0x1a18288, 0x1a18287, 0xc41ff81889, 0x0)
    /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc42110f2e0, 0x233c140, 0x2835730, 0xc4212092c0, 0x0, 0x0, 0x17b32e0, 0xc420fcee80, 0x7fffffff, 0xc420bd7b01, ...)
    /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*serverStream).RecvMsg(0xc420b96cf0, 0x17b32e0, 0xc420fcee80, 0x0, 0x0)
    /go/src/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc420c72a70, 0xcb6dbc, 0x16ea6a0, 0xc4201a8030)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:228 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(Gossip_GossipServer).Recv-fm(0xc4201a8030, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:153 +0x2f
github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc4201a8000, 0x7f3dd1216be8, 0xc420fcf480, 0xc4203c20d0, 0xc420fcf4c0, 0xc420bd7f40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:321 +0x413
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:153 +0x99
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201ba090, 0xc4203b9180)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 406 [select, 9 minutes]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc421976750)
    /go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c
created by google.golang.org/grpc/transport.newHTTP2Server
    /go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767

goroutine 458 [IO wait]:
net.runtime_pollWait(0x7f3dd1265000, 0x72, 0x1a)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc421a33870, 0x72, 0xc421d0d598, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc421a33870, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc421a33810, 0xc421768400, 0x400, 0x400, 0x0, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420176380, 0xc421768400, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420bdd760, 0xc421768400, 0x400, 0x400, 0x17b3040, 0x0, 0x7f3dd009f040)
    /go/src/github.com/cockroachdb/cmux/cmux.go:218 +0x113
crypto/tls.(*block).readFromUntil(0xc420ff4600, 0x7f3dd009f040, 0xc420bdd760, 0x5, 0xc420bdd760, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc42168a380, 0x1a20017, 0xc42168a488, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc42168a380, 0xc4219ee000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc420897440)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420897440, 0xc4203e4638, 0x9, 0x9, 0x60c170, 0xc42900b2c0, 0x0)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x2325a80, 0xc420897440, 0xc4203e4638, 0x9, 0x9, 0x9, 0xc421d0dae8, 0x620340, 0xc423db7780)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x2325a80, 0xc420897440, 0xc4203e4638, 0x9, 0x9, 0xc42900b2c0, 0x0, 0x1)
    /usr/local/go/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc4203e4638, 0x9, 0x9, 0x2325a80, 0xc420897440, 0x0, 0xc400000000, 0xc421d0dc80, 0xc45649)
    /go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc4203e4600, 0x31, 0x31, 0x0, 0x0)
    /go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc420ff4d20, 0xc428cdd470, 0xc428cdd470, 0x0, 0x0)
    /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc421976750, 0xc420ff4de0)
    /go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200
google.golang.org/grpc.(*Server).serveStreams(0xc420b36640, 0x2343b40, 0xc421976750)
    /go/src/google.golang.org/grpc/server.go:424 +0x160
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc420b36640, 0x2343f60, 0xc42168a380, 0x232d980, 0xc42001b600)
    /go/src/google.golang.org/grpc/server.go:411 +0x3da
google.golang.org/grpc.(*Server).handleRawConn(0xc420b36640, 0x2343fc0, 0xc420bdd760)
    /go/src/google.golang.org/grpc/server.go:388 +0x470
created by google.golang.org/grpc.(*Server).Serve
    /go/src/google.golang.org/grpc/server.go:357 +0x1a9

goroutine 479 [select, 9 minutes]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420f62e00, 0xc4216c8110, 0x5, 0x5, 0x0, 0x0, 0x0)
    /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc420ca24b0, 0xc4216c8110, 0x5, 0x5, 0x8000, 0x9, 0x6fdb99)
    /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x232a7c0, 0xc420ca24b0, 0xc4216c8110, 0x5, 0x5, 0x5, 0x0, 0x166dda0, 0xc420cf7cc0)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x232a7c0, 0xc420ca24b0, 0xc4216c8110, 0x5, 0x5, 0xc420dd1500, 0xc420cf7d70, 0x0)
    /usr/local/go/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc4216c8100, 0x7fffffff, 0x0, 0x0, 0xc420f62e48, 0x0, 0xc420cf7f18, 0x0)
    /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc4216c8100, 0x233c140, 0x2835730, 0xc420ca24b0, 0x0, 0x0, 0x17a7040, 0xc4201626d8, 0x7fffffff, 0xc420034800, ...)
    /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*clientStream).RecvMsg(0xc421c3e480, 0x17a7040, 0xc4201626d8, 0x0, 0x0)
    /go/src/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*distSQLFlowStreamClient).CloseAndRecv(0xc4211a9eb0, 0x0, 0x0, 0xaeae03)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/api.pb.go:201 +0x8b
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*outbox).run(0xc4203d8960)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/outbox.go:185 +0x194
created by github.com/cockroachdb/cockroach/pkg/sql/distsql.(*outbox).start
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/outbox.go:203 +0x70

goroutine 498 [select, 9 minutes]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc420519100)
    /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d
created by google.golang.org/grpc/transport.newHTTP2Client
    /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78

goroutine 333 [chan receive, 9 minutes]:
database/sql.(*DB).connectionOpener(0xc421506420)
    /usr/local/go/src/database/sql/sql.go:730 +0x4a
created by database/sql.Open
    /usr/local/go/src/database/sql/sql.go:493 +0x1e9

goroutine 370 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:650 +0x2a9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420e9ede0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 371 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:609 +0x4f0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420e9ee00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 351 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420266c80, 0xc420527100, 0xc4211d9920, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:292 +0x617
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:227 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201ba090, 0xc421070060)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 373 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:144 +0x465
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420e9eea0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 374 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).startSampleEnvironment.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:768 +0x16b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420e9eec0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 375 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:90 +0x16f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc42112c5b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 376 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteSummaries.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:688 +0x26d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420c871a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 377 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:751 +0x12b3
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420e9eee0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 378 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*flowScheduler).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/flow_scheduler.go:101 +0x314
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc42112c5c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 387 [select, 9 minutes]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc42165c400)
    /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d
created by google.golang.org/grpc/transport.newHTTP2Client
    /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78

goroutine 386 [IO wait]:
net.runtime_pollWait(0x7f3dd1298688, 0x72, 0x13)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42109bbf0, 0x72, 0xc420cc1828, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42109bbf0, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42109bb90, 0xc420529800, 0x400, 0x400, 0x0, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420176260, 0xc420529800, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc421110420, 0x7f3dd121b258, 0xc420176260, 0x5, 0xc420176260, 0xc421700124)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc42154a380, 0x1a20017, 0xc42154a488, 0xc421e867f0)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc42154a380, 0xc42169e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc420c439e0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420c439e0, 0xc4203e4878, 0x9, 0x9, 0xc420cc1ca8, 0x620340, 0xc42632ff00)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x2325a80, 0xc420c439e0, 0xc4203e4878, 0x9, 0x9, 0x9, 0xc4231d16f0, 0xc4231d16e8, 0x232a900)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x2325a80, 0xc420c439e0, 0xc4203e4878, 0x9, 0x9, 0xc420cc1eb0, 0xc41300, 0xc42770c0f0)
    /usr/local/go/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc4203e4878, 0x9, 0x9, 0x2325a80, 0xc420c439e0, 0x0, 0xc400000000, 0xc421700124, 0xc)
    /go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc4203e4840, 0x0, 0x0, 0x0, 0x0)
    /go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc421110ed0, 0xc4231d1b30, 0xc4231d1b30, 0x0, 0x0)
    /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Client).reader(0xc42165c400)
    /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6
created by google.golang.org/grpc/transport.newHTTP2Client
    /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1

goroutine 141 [select, 9 minutes]:
google.golang.org/grpc.NewClientStream.func2(0x2343ae0, 0xc42165c400, 0xc4200a61e0, 0xc421852000)
    /go/src/google.golang.org/grpc/stream.go:197 +0x426
created by google.golang.org/grpc.NewClientStream
    /go/src/google.golang.org/grpc/stream.go:217 +0xb5f

goroutine 425 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:80 +0x1ed
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421690d40)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 426 [chan receive, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:171 +0x55
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421690f80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 427 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe00000000)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691080)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 428 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe0000000c)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4216910a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 429 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe0000000d)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691140)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 430 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe00000005)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4216911a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 431 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe00000003)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691200)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 432 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe0000000e)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691220)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 433 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe0000000f)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691240)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 434 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe0000000a)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691260)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 435 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe00000008)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691280)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 436 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe00000009)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4216912a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 437 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe00000004)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4216912c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 438 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe00000007)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4216912e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 439 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe0000000b)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691300)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 440 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe00000006)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691320)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 441 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe00000002)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691340)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 442 [semacquire, 9 minutes]:
sync.runtime_notifyListWait(0xc421100f50, 0xe00000001)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421100f40)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200a4aa0, 0xc4201bb320)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691360)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 443 [semacquire, 9 minutes]:
sync.runtime_Semacquire(0xc4200a4b3c)
    /usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc4200a4b30)
    /usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc4200a4aa0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:187 +0x34
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaft.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3258 +0x3a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4211d94d0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 444 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3284 +0x334
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4211d94e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 445 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startCoalescedHeartbeatsLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3315 +0x167
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4211d94f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 447 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func3()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1149 +0x165
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4216913e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 448 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1245 +0x39b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421911cb0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 449 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1245 +0x39b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421911ce0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 346 [IO wait, 9 minutes]:
net.runtime_pollWait(0x7f3dd1298508, 0x72, 0x17)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420d7aed0, 0x72, 0xc42192d068, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420d7aed0, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420d7ae70, 0xc421bd3000, 0x800, 0x800, 0x0, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4207720b0, 0xc421bd3000, 0x800, 0x800, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4206fc580, 0xc421bd3000, 0x800, 0x800, 0x17b3040, 0x0, 0x7f3dd009f040)
    /go/src/github.com/cockroachdb/cmux/cmux.go:218 +0x113
crypto/tls.(*block).readFromUntil(0xc421096690, 0x7f3dd009f040, 0xc4206fc580, 0x5, 0xc4206fc580, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc420d40380, 0x1a20017, 0xc420d40488, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc420d40380, 0xc420636000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc4211d09c0)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).ReadByte(0xc4211d09c0, 0x0, 0xc42192d500, 0xa8e6ab)
    /usr/local/go/src/bufio/bufio.go:231 +0x51
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readBuffer).readTypedMsg(0xc420242028, 0x23335c0, 0xc4211d09c0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/encoding.go:94 +0x35
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*v3Conn).serve(0xc420242000, 0x7f3dd1216800, 0xc421192e70, 0x5400, 0xc4203d1d38, 0x7f3dd1216800, 0xc4207c7860, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:324 +0x344
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc4203d1c00, 0x7f3dd1216800, 0xc4207c7860, 0x2343f60, 0xc420d40380, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:309 +0x999
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func8.1(0x2343fc0, 0xc4206fc580)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:502 +0x106
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4201ba090, 0xc420032058, 0x2343fc0, 0xc4206fc580, 0xc4202b22a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:136 +0x95
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
    /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:138 +0x277

goroutine 452 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691400)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 453 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc4200a4d20, 0x7f3dd1216800, 0xc421911d10, 0xecfc4be94, 0xc436035910, 0x27f8b00, 0xc420f52c00, 0xc4201bb320, 0x0, 0xc4200a4ab8)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x254
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x3ee
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc421691420)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 389 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420ffa2a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 390 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420ffa2e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 391 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420ffa320)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 392 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420ffa360)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 393 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420ffa3a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 394 [select, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420ffa3c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 395 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc4200a4b40, 0x7f3dd1216800, 0xc4207c6d50, 0xecfc4c0eb, 0xc42ebf5aa4, 0x27f8b00, 0xc420f52c00, 0xc4201bb320, 0x0, 0xc48600)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x254
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x3ee
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc420ffa3e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 474 [select, 9 minutes]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc4215a4280)
    /go/src/google.golang.org/grpc/clientconn.go:720 +0x63e
google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4215a4280)
    /go/src/google.golang.org/grpc/clientconn.go:479 +0x1de
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/google.golang.org/grpc/clientconn.go:480 +0x3c7

goroutine 478 [IO wait]:
net.runtime_pollWait(0x7f3dd1298388, 0x72, 0x18)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42005ff00, 0x72, 0xc421d0f598, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42005ff00, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42005fea0, 0xc421443000, 0x400, 0x400, 0x0, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420772170, 0xc421443000, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4206fcf20, 0xc421443000, 0x400, 0x400, 0x17b3040, 0x0, 0x7f3dd009f040)
    /go/src/github.com/cockroachdb/cmux/cmux.go:218 +0x113
crypto/tls.(*block).readFromUntil(0xc4210961b0, 0x7f3dd009f040, 0xc4206fcf20, 0x5, 0xc4206fcf20, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc42154ae00, 0x1a20017, 0xc42154af08, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc42154ae00, 0xc420f84000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc42105b080)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc42105b080, 0xc4201a8ff8, 0x9, 0x9, 0x60c170, 0xc428ebdf30, 0x0)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x2325a80, 0xc42105b080, 0xc4201a8ff8, 0x9, 0x9, 0x9, 0xc421d0fae8, 0x620340, 0xc421b34500)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x2325a80, 0xc42105b080, 0xc4201a8ff8, 0x9, 0x9, 0xc428ebdf30, 0x0, 0x1)
    /usr/local/go/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc4201a8ff8, 0x9, 0x9, 0x2325a80, 0xc42105b080, 0x0, 0xc400000000, 0xc421d0fc80, 0xc45649)
    /go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc4201a8fc0, 0x31, 0x31, 0x0, 0x0)
    /go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc421071740, 0xc4287b8360, 0xc4287b8360, 0x0, 0x0)
    /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4201bb170, 0xc421071830)
    /go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200
google.golang.org/grpc.(*Server).serveStreams(0xc420b36640, 0x2343b40, 0xc4201bb170)
    /go/src/google.golang.org/grpc/server.go:424 +0x160
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc420b36640, 0x2343f60, 0xc42154ae00, 0x232d980, 0xc4203fed10)
    /go/src/google.golang.org/grpc/server.go:411 +0x3da
google.golang.org/grpc.(*Server).handleRawConn(0xc420b36640, 0x2343fc0, 0xc4206fcf20)
    /go/src/google.golang.org/grpc/server.go:388 +0x470
created by google.golang.org/grpc.(*Server).Serve
    /go/src/google.golang.org/grpc/server.go:357 +0x1a9

goroutine 481 [semacquire, 9 minutes]:
sync.runtime_Semacquire(0xc42160b53c)
    /usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc42160b530)
    /usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*Flow).Wait(0xc42160b4a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/flow.go:351 +0x34
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*flowScheduler).runFlowNow.func2(0xc42160b4a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/flow_scheduler.go:68 +0x2b
created by github.com/cockroachdb/cockroach/pkg/sql/distsql.(*flowScheduler).runFlowNow
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/flow_scheduler.go:70 +0xb3

goroutine 142 [semacquire, 9 minutes]:
sync.runtime_Semacquire(0xc4202b609c)
    /usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc4202b6090)
    /usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*Flow).Wait(0xc4202b6000)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/flow.go:351 +0x34
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*ServerImpl).RunSyncFlow.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/server.go:133 +0x93
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc4201bb320, 0xc420e19c20, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:224 +0x100
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*ServerImpl).RunSyncFlow(0xc420eb6000, 0xc4200a6780, 0x2341500, 0xc420d604c0, 0x0, 0x5dbde8)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/server.go:135 +0x198
github.com/cockroachdb/cockroach/pkg/sql/distsql._DistSQL_RunSyncFlow_Handler(0x18118e0, 0xc420eb6000, 0x233f8e0, 0xc421c50000, 0xc42161bd18, 0x5f8f0a)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/api.pb.go:232 +0x117
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420b36640, 0x2343b40, 0xc420407ef0, 0xc4200a65a0, 0xc4200ca6c0, 0x23170e0, 0x0, 0x0, 0x0)
    /go/src/google.golang.org/grpc/server.go:689 +0x702
google.golang.org/grpc.(*Server).handleStream(0xc420b36640, 0x2343b40, 0xc420407ef0, 0xc4200a65a0, 0x0)
    /go/src/google.golang.org/grpc/server.go:773 +0xc36
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42112d600, 0xc420b36640, 0x2343b40, 0xc420407ef0, 0xc4200a65a0)
    /go/src/google.golang.org/grpc/server.go:422 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
    /go/src/google.golang.org/grpc/server.go:423 +0xa3

goroutine 144 [chan send, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*RowChannel).PushRow(0xc4203c45a0, 0xc420d30300, 0x2, 0x2, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/base.go:154 +0xc7
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*tableReader).Run(0xc4219d8000, 0xc4202b6090)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/tablereader.go:129 +0x2c4
created by github.com/cockroachdb/cockroach/pkg/sql/distsql.(*Flow).Start
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/flow.go:345 +0x2cb

goroutine 517 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420267780, 0xc42015cb00, 0xc42112dc70, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:292 +0x617
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:227 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205e4e10, 0xc420ff8660)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 516 [select, 9 minutes]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc420700780)
    /go/src/google.golang.org/grpc/clientconn.go:720 +0x63e
google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420700780)
    /go/src/google.golang.org/grpc/clientconn.go:479 +0x1de
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/google.golang.org/grpc/clientconn.go:480 +0x3c7

goroutine 483 [IO wait]:
net.runtime_pollWait(0x7f3dd12982c8, 0x72, 0x19)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420de8290, 0x72, 0xc4211e1828, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420de8290, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420de8230, 0xc421998400, 0x400, 0x400, 0x0, 0x232e740, 0xc420014240)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420db2010, 0xc421998400, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc42101b230, 0x7f3dd121b258, 0xc420db2010, 0x5, 0xc420db2010, 0xc42106c504)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc420562000, 0x1a20017, 0xc420562108, 0xc42900b330)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc420562000, 0xc42172e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1113 +0x116
bufio.(*Reader).fill(0xc4210cf080)
    /usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc4210cf080, 0xc4207c9bf8, 0x9, 0x9, 0xc4211e1ca8, 0x620340, 0xc4215198c0)
    /usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x2325a80, 0xc4210cf080, 0xc4207c9bf8, 0x9, 0x9, 0x9, 0xc428cdd120, 0xc428cdd118, 0x232a900)
    /usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x2325a80, 0xc4210cf080, 0xc4207c9bf8, 0x9, 0x9, 0xc4211e1eb0, 0xc41300, 0xc42909e870)
    /usr/local/go/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc4207c9bf8, 0x9, 0x9, 0x2325a80, 0xc4210cf080, 0x0, 0xc400000000, 0xc42106c504, 0xc)
    /go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc4207c9bc0, 0x0, 0x0, 0x0, 0x0)
    /go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc421110f90, 0xc428cdd6b0, 0xc428cdd6b0, 0x0, 0x0)
    /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Client).reader(0xc421501300)
    /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6
created by google.golang.org/grpc/transport.newHTTP2Client
    /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1

goroutine 485 [select, 9 minutes]:
google.golang.org/grpc.NewClientStream.func2(0x2343ae0, 0xc421501300, 0xc420ca24b0, 0xc421c3e480)
    /go/src/google.golang.org/grpc/stream.go:197 +0x426
created by google.golang.org/grpc.NewClientStream
    /go/src/google.golang.org/grpc/stream.go:217 +0xb5f

goroutine 486 [chan send, 9 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*RowChannel).PushRow(0xc4203c4560, 0xc420f5a300, 0x2, 0x2, 0xc420f5a300)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/base.go:154 +0xc7
github.com/cockroachdb/cockroach/pkg/sql/distsql.ProcessInboundStream(0xc4202b6000, 0x2343600, 0xc420c26090, 0x0, 0x232fa00, 0xc4203c4560, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/inbound.go:82 +0x228
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*ServerImpl).flowStreamInt(0xc420eb6000, 0x2343600, 0xc420c26090, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/server.go:174 +0x251
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*ServerImpl).FlowStream(0xc420eb6000, 0x2343600, 0xc420c26090, 0xc4207b0ca8, 0x20)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/server.go:180 +0x8f
github.com/cockroachdb/cockroach/pkg/sql/distsql._DistSQL_FlowStream_Handler(0x18118e0, 0xc420eb6000, 0x233f8e0, 0xc420e82240, 0x8000, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql/api.pb.go:267 +0xbb
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420b36640, 0x2343b40, 0xc421976750, 0xc420ca2c30, 0xc4200ca6c0, 0x2317100, 0x0, 0x0, 0x0)
    /go/src/google.golang.org/grpc/server.go:689 +0x702
google.golang.org/grpc.(*Server).handleStream(0xc420b36640, 0x2343b40, 0xc421976750, 0xc420ca2c30, 0x0)
    /go/src/google.golang.org/grpc/server.go:773 +0xc36
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420c73060, 0xc420b36640, 0x2343b40, 0xc421976750, 0xc420ca2c30)
    /go/src/google.golang.org/grpc/server.go:422 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
    /go/src/google.golang.org/grpc/server.go:423 +0xa3

rax    0xca
rbx    0x27fa150
rcx    0xffffffffffffffff
rdx    0x0
rdi    0x27fae50
rsi    0x0
rbp    0x7fffa3921ec0
rsp    0x7fffa3921e78
r8     0x0
r9     0x0
r10    0x0
r11    0x286
r12    0x10
r13    0x12f0
r14    0x14f
r15    0x0
rip    0x62a681
rflags 0x286
cs     0x33
fs     0x0
gs     0x0

ERROR: exit status 2

168 runs completed, 1 failures, over 11m18s
FAIL
Makefile:128: recipe for target 'stress' failed
make: *** [stress] Error 1
jordanlewis commented 7 years ago

SIGABRT during TestClusterFlow.