github.com/cockroachdb/cockroach/pkg/kv: TestMultiRangeBoundedBatchDelRangeBoundary failed under stress

SHA: https://github.com/cockroachdb/cockroach/commits/5b28afe7d1eeb4163278d0202de7b752b0210178

Parameters:

COCKROACH_PROPOSER_EVALUATED_KV=true
TAGS=
GOFLAGS=

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=107245&tab=buildLog

W170104 07:32:09.470576 18542 server/status/runtime.go:116  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
I170104 07:32:09.471438 18542 server/config.go:447  1 storage engine initialized
I170104 07:32:09.471731 18542 server/node.go:426  [n?] store [n0,s0] not bootstrapped
I170104 07:32:09.473724 18542 server/node.go:355  [n?] **** cluster 740eecb1-102e-46af-b162-839eba98bcbf has been created
I170104 07:32:09.473759 18542 server/node.go:356  [n?] **** add additional nodes by specifying --join=127.0.0.1:60763
I170104 07:32:09.475590 18542 storage/store.go:1258  [n1] [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170104 07:32:09.475616 18542 server/node.go:439  [n1] initialized store [n1,s1]: {Capacity:536870912 Available:536870912 RangeCount:1 LeaseCount:0}
I170104 07:32:09.476760 18542 server/node.go:324  [n1] node ID 1 initialized
I170104 07:32:09.476799 18542 gossip/gossip.go:292  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:60763" > attrs:<> locality:<> 
I170104 07:32:09.476943 18987 storage/split_queue.go:101  [split,n1,s1,r1/1:/M{in-ax},@c4207a2d80] splitting at keys [/Table/11/0 /Table/12/0 /Table/13/0 /Table/14/0]
I170104 07:32:09.477113 18542 storage/stores.go:296  [n1] read 0 node addresses from persistent storage
I170104 07:32:09.477158 18542 server/node.go:569  [n1] connecting to gossip network to verify cluster ID...
I170104 07:32:09.477174 18542 server/node.go:589  [n1] node connected via gossip and verified as part of cluster "740eecb1-102e-46af-b162-839eba98bcbf"
I170104 07:32:09.477278 18542 server/node.go:374  [n1] node=1: started with [[]=] engine(s) and attributes []
I170104 07:32:09.477300 18542 sql/executor.go:322  [n1] creating distSQLPlanner with address {tcp 127.0.0.1:60763}
I170104 07:32:09.479808 19074 sql/event_log.go:95  [n1] Event: "node_join", target: 1, info: {Descriptor:{NodeID:1 Address:{NetworkField:tcp AddressField:127.0.0.1:60763} Attrs: Locality:} ClusterID:740eecb1-102e-46af-b162-839eba98bcbf StartedAt:1483515129477263285}
I170104 07:32:09.480482 18987 storage/replica_command.go:2362  [split,n1,s1,r1/1:/M{in-ax},@c4207a2d80] initiating a split of this range at key /Table/11 [r2]
I170104 07:32:09.483337 18542 server/server.go:621  [n1] starting https server at 127.0.0.1:48939
I170104 07:32:09.483353 18542 server/server.go:622  [n1] starting grpc/postgres server at 127.0.0.1:60763
I170104 07:32:09.483359 18542 server/server.go:623  [n1] advertising CockroachDB node at 127.0.0.1:60763
W170104 07:32:09.115943 19076 util/hlc/hlc.go:145  backward time jump detected (-0.367466 seconds)
E170104 07:32:09.116399 18987 internal/client/txn.go:333  [split,n1,s1,r1/1:/M{in-ax},@c4207a2d80] failure aborting transaction: storage/store.go:2368: rejecting command with timestamp in the future: 1483515129480521874 (364.186558ms ahead); abort caused by: storage/store.go:2368: rejecting command with timestamp in the future: 1483515129480521874 (364.35673ms ahead)
E170104 07:32:09.116447 18987 storage/queue.go:597  [split,n1,s1,r1/1:/M{in-ax},@c4207a2d80] unable to split [n1,s1,r1/1:/M{in-ax}] at key "/Table/11/0": storage/replica_command.go:2441: split at key /Table/11 failed: storage/store.go:2368: rejecting command with timestamp in the future: 1483515129480521874 (364.35673ms ahead)
E170104 07:32:09.116877 19082 storage/node_liveness.go:187  [n1,hb] failed liveness heartbeat: storage/store.go:2368: rejecting command with timestamp in the future: 1483515129481530103 (364.735291ms ahead)
W170104 07:32:09.117466 19076 ts/db.go:113  [n1,ts-poll] error writing time series data: storage/store.go:2368: rejecting command with timestamp in the future: 1483515129481530103 (364.2871ms ahead)
F170104 07:32:09.118253 18542 server/server.go:674  [n1] failed to get list of completed migrations: storage/store.go:2368: rejecting command with timestamp in the future: 1483515129481530103 (363.377812ms ahead)
goroutine 18542 [running]:
github.com/cockroachdb/cockroach/pkg/util/log.getStacks(0x4801, 0x70c690a, 0x2a26ae0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:849 +0xa5
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).outputLogEntry(0x2a28120, 0xc400000004, 0x2243902, 0x10, 0x2a2, 0xc422d10500, 0xa0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:714 +0x7b1
github.com/cockroachdb/cockroach/pkg/util/log.addStructured(0x2ae3593c3b80, 0xc421d36a50, 0x4, 0x2, 0x0, 0x0, 0xc4229e53a8, 0x1, 0x1)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/structured.go:146 +0x296
github.com/cockroachdb/cockroach/pkg/util/log.logDepth(0x2ae3593c3b80, 0xc421d36a50, 0x1, 0xc400000004, 0x0, 0x0, 0xc4229e53a8, 0x1, 0x1)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:88 +0x8c
github.com/cockroachdb/cockroach/pkg/util/log.Fatal(0x2ae3593c3b80, 0xc421d36a50, 0xc4229e53a8, 0x1, 0x1)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:181 +0x76
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start(0xc420b19400, 0x2ae3593c3b80, 0xc421d36a50, 0x2ae359405100, 0xc420014568)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:674 +0x2140
github.com/cockroachdb/cockroach/pkg/server.(*TestServer).Start(0xc42231f100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/testserver.go:279 +0x184
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerRaw(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:161 +0x134
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServer(0x256d800, 0xc420ada600, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:131 +0x63
github.com/cockroachdb/cockroach/pkg/kv_test.TestMultiRangeBoundedBatchDelRangeBoundary(0xc420ada600)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender_server_test.go:530 +0xd3
testing.tRunner(0xc420ada600, 0x1b88468)
    /usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
    /usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42015e480, 0x19cfe41, 0x2a, 0x1b88468, 0xc42026fc01)
    /usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42015e480)
    /usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42015e480, 0xc42012ee00)
    /usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x1b8b860, 0x25450e0, 0x89, 0x89, 0x2a639b0)
    /usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc42012eef8, 0x2a634c0)
    /usr/local/go/src/testing/testing.go:743 +0x85
github.com/cockroachdb/cockroach/pkg/kv_test.TestMain(0xc42012eef8)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/main_test.go:53 +0x76
main.main()
    github.com/cockroachdb/cockroach/pkg/kv/_test/_testmain.go:330 +0xc6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 20 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).flushDaemon(0x2a28120)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1016 +0x77
created by github.com/cockroachdb/cockroach/pkg/util/log.init.1
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:581 +0x95

goroutine 18906 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials.(*tlsCreds).ClientHandshake(0xc4201243d0, 0x2ae359383638, 0xc422cf4960, 0xc422ebe010, 0xf, 0x256a8a0, 0xc421aa81b8, 0x0, 0x0, 0x1799880, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials/credentials.go:160 +0x329
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client(0x2ae359383638, 0xc422cf4960, 0xc422ebe010, 0xf, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:164 +0x141d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.NewClientTransport(0x2ae359383638, 0xc422cf4960, 0xc422ebe010, 0xf, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:385 +0xa1
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc4224c66e0, 0xc420359700, 0x0, 0x649d71)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:691 +0x276
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4224c66e0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:498 +0x34
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:507 +0x3c3

goroutine 19042 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe0000000b)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68500)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19031 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xc400000011)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68240)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18987 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x383
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421dc3660)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19005 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1033 +0x394
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421eb7bb0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19045 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe0000000e)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f685a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18997 [IO wait]:
net.runtime_pollWait(0x2ae359408e98, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42074f1e0, 0x72, 0xc42039de08, 0xc420014220)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42074f1e0, 0x2552c80, 0xc420014220)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc42074f180, 0x0, 0x254fa00, 0xc422c6fcc0)
    /usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc4218401a8, 0xc42039def0, 0xc42039df00, 0x6409d0)
    /usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc4218401a8, 0x1b8a1e0, 0xc4223b1e80, 0xc422c66d10, 0xc420bf6f50)
    /usr/local/go/src/net/tcpsock.go:222 +0x49
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc4223b1e80, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0xa4
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func3()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:483 +0x2f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421faa740)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18995 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:98 +0x70
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421faa420)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19075 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).startSampleEnvironment.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:806 +0x16b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68a80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19040 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe0000000a)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68480)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19034 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xc400000014)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f682c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19002 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func8()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:514 +0x295
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421d36d20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19006 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func1(0xc42230ee60)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:375 +0x44
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:376 +0x702

goroutine 18994 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:98 +0x70
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421faa3e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18967 [IO wait]:
net.runtime_pollWait(0x2ae3594090d8, 0x72, 0x8)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420902d10, 0x72, 0xc4216e9388, 0xc420014220)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420902d10, 0x2552c80, 0xc420014220)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420902cb0, 0xc42089d400, 0x400, 0x400, 0x0, 0x2552c80, 0xc420014220)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4212da1d8, 0xc42089d400, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420d22b00, 0xc42089d400, 0x400, 0x400, 0x18f5640, 0x0, 0x2ae35ca12f68)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x113
crypto/tls.(*block).readFromUntil(0xc4223b5800, 0x2ae35ca12f68, 0xc420d22b00, 0x5, 0xc420d22b00, 0x1)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc42149ce00, 0x516, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).readHandshake(0xc42149ce00, 0x539, 0x0, 0x0, 0x4)
    /usr/local/go/src/crypto/tls/conn.go:929 +0x8f
crypto/tls.(*serverHandshakeState).doFullHandshake(0xc4216e9b08, 0xc422bc9b00, 0x0)
    /usr/local/go/src/crypto/tls/handshake_server.go:447 +0x5a2
crypto/tls.(*Conn).serverHandshake(0xc42149ce00, 0x1b8c1b8, 0xc42149cf08)
    /usr/local/go/src/crypto/tls/handshake_server.go:86 +0x30d
crypto/tls.(*Conn).Handshake(0xc42149ce00, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1262 +0x2c3
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials.(*tlsCreds).ServerHandshake(0xc421840140, 0x256a780, 0xc420d22b00, 0x1, 0x1, 0x2560380, 0xc421840190, 0xc422317e00, 0xc421840198)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials/credentials.go:175 +0xd2
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).useTransportAuthenticator(0xc420ada6c0, 0x256a780, 0xc420d22b00, 0xc4214fadc0, 0xc4212da1c8, 0xc4214fae40, 0xc4214fae38, 0xc42212e768, 0x86f03f)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:324 +0x50
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc420ada6c0, 0x256a780, 0xc420d22b00)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:391 +0x5a
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:384 +0x5ac

goroutine 18999 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc421faa720, 0xc4218401a8, 0xc422cae570, 0x17f8020, 0x252e850)
    <autogenerated>:9 +0x7f
crypto/tls.(*listener).Accept(0xc421faa7a0, 0xc422cae540, 0x17f8020, 0x252e850, 0x18a0c20)
    /usr/local/go/src/crypto/tls/tls.go:52 +0x37
net/http.(*Server).Serve(0xc422286f80, 0x2558040, 0xc421faa7a0, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:2273 +0x1ce
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:494 +0x42
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421faa840)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19058 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*idAllocator).start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/id_alloc.go:133 +0x7c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc422ee8890)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19047 [semacquire]:
sync.runtime_Semacquire(0xc420d5d5f4)
    /usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc420d5d5e8)
    /usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420d5d550)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:187 +0x34
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaft.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3406 +0x3a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc42228f890)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19035 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe00000005)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68300)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19043 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe0000000c)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68540)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18993 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc4214fbb80, 0x2ae3593c3b80, 0xc421d2f140, 0xecffe9bf9, 0x1c6be5b1, 0x2a26ae0, 0xc4223b1740, 0xc420370480, 0xc4207a2d80, 0xecffe9bf9)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x254
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc4207a2d80, 0xc421d18390)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:269 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc421d18390, 0xc421e9b040)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:339 +0x303
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:271 +0x2ca
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421dc3720)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19056 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:657 +0x2bd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68960)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19053 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1313 +0x39b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421d18de0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19033 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xc400000013)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f682a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18996 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:471 +0x71
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421d36cc0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18988 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x383
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421dc3680)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18545 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1174 +0xccb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc422323f20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18998 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc421faa700, 0xc422345500, 0x17f8020, 0x252e850, 0x18a0c20)
    <autogenerated>:9 +0x7f
net/http.(*Server).Serve(0xc422287000, 0x2560cc0, 0xc421faa700, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:2273 +0x1ce
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func4()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:487 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421faa760)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19030 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:171 +0x55
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68220)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19051 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func3()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1227 +0x165
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68640)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19036 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe00000006)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68320)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19039 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe00000009)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f683e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19001 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc421faa5a0, 0x1b8b578, 0xc420ada6c0, 0x256a780, 0xc420d22b00)
    <autogenerated>:9 +0x7f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc420ada6c0, 0x2560cc0, 0xc421faa5a0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:354 +0x187
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:505 +0x43
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421faa8e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18990 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x383
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421dc36c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18544 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).printStatsLoop(0xc4214fb220, 0x2ae3593c3b80, 0xc422cae510)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:209 +0xbd3
github.com/cockroachdb/cockroach/pkg/kv.NewTxnCoordSender.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:193 +0x98
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421eb7190)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18543 [chan receive]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421fd7340)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19052 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1313 +0x39b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421d18db0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19000 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:498 +0x6d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421faa880)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19004 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:995 +0x4c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421eb7ba0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18991 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x383
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421dc36e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18989 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x383
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421dc36a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19003 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:381 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421faaa00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19032 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xc400000012)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68260)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18907 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).wait(0xc4224c66e0, 0x2ae359383638, 0xc4223342a0, 0x100, 0xf, 0x48, 0x1b831f9, 0x1b831f8)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:840 +0x183
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).getTransport(0xc4208b4240, 0x2ae359383638, 0xc4223342a0, 0x0, 0x0, 0xc421d12050, 0x1b831ad, 0xc422352d40, 0x1abb7e33690)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:554 +0x2b4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x2ae359383638, 0xc4223342a0, 0x19bb7be, 0x1d, 0x18f21c0, 0xc422352d00, 0x18e0a60, 0xc421da81e0, 0xc4208b4240, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:173 +0x2bf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x2ae359383638, 0xc4223342a0, 0x19bb7be, 0x1d, 0x18f21c0, 0xc422352d00, 0x18e0a60, 0xc421da81e0, 0xc4208b4240, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:118 +0x19c
github.com/cockroachdb/cockroach/pkg/rpc.(*heartbeatClient).Ping(0xc4212da1a8, 0x2ae359383638, 0xc4223342a0, 0xc422352d00, 0x0, 0x0, 0x0, 0xc4223b1750, 0xc4223b1750, 0xc420129d38)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/heartbeat.pb.go:121 +0xd2
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).heartbeat(0xc422286d80, 0x254a880, 0xc4212da1a8, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421eb7ac0, 0xf, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:343 +0x15d
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc422286d80, 0xc4208b4240, 0xc422ebe010, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:299 +0x291
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421d24510)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19041 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xc400000010)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f684a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18986 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x383
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421dc3620)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19037 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe00000007)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68360)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19057 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:616 +0x4f0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f689a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18992 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x383
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421dc3700)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19049 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startCoalescedHeartbeatsLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3463 +0x167
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc42228f8b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 18923 [runnable]:
math/big.putNat(0xc422b89440, 0x21, 0x24)
    /usr/local/go/src/math/big/nat.go:554
math/big.nat.divLarge(0xc42132e900, 0x20, 0x44, 0xc42132e240, 0x40, 0x45, 0xc42132e000, 0x40, 0x44, 0xc42106fa40, ...)
    /usr/local/go/src/math/big/nat.go:634 +0x645
math/big.nat.div(0xc42132e900, 0x20, 0x44, 0xc42132e240, 0x40, 0x45, 0xc42132e000, 0x40, 0x44, 0xc42106fa40, ...)
    /usr/local/go/src/math/big/nat.go:541 +0x3cb
math/big.nat.expNN(0x0, 0x0, 0x0, 0xc420142240, 0x20, 0x24, 0xc42233c690, 0x1, 0x5, 0xc42106fa40, ...)
    /usr/local/go/src/math/big/nat.go:985 +0x41b
math/big.(*Int).Exp(0xc422cb70f8, 0xc422cb7118, 0xc421a2d480, 0xc421fd6d60, 0x24)
    /usr/local/go/src/math/big/int.go:420 +0xf2
crypto/rsa.encrypt(0xc422cb70f8, 0xc421eb6b70, 0xc422cb7118, 0x131)
    /usr/local/go/src/crypto/rsa/rsa.go:331 +0x5e
crypto/rsa.VerifyPKCS1v15(0xc421eb6b70, 0x5, 0xc421a2d360, 0x20, 0x20, 0xc4207e424f, 0x100, 0x131, 0xc421a2d300, 0xc421eea648)
    /usr/local/go/src/crypto/rsa/pkcs1v15.go:277 +0x197
crypto/x509.checkSignature(0x4, 0xc4207e400e, 0x22d, 0x372, 0xc4207e424f, 0x100, 0x131, 0x175df40, 0xc421eb6b70, 0x0, ...)
    /usr/local/go/src/crypto/x509/x509.go:706 +0x299
crypto/x509.(*Certificate).CheckSignature(0xc420099200, 0x4, 0xc4207e400e, 0x22d, 0x372, 0xc4207e424f, 0x100, 0x131, 0xc421d9a550, 0xc421298840)
    /usr/local/go/src/crypto/x509/x509.go:673 +0x97
crypto/x509.(*Certificate).CheckSignatureFrom(0xc42068a480, 0xc420099200, 0xc4207e4039, 0x2d)
    /usr/local/go/src/crypto/x509/x509.go:667 +0xd7
crypto/x509.(*CertPool).findVerifiedParents(0xc422322db0, 0xc42068a480, 0xc422cb7408, 0x941d71, 0xc421eea6d0, 0x2547e10, 0xc, 0xc421eea6d0)
    /usr/local/go/src/crypto/x509/cert_pool.go:57 +0xe3
crypto/x509.(*Certificate).buildChains(0xc42068a480, 0xc422cb7638, 0xc422cb7610, 0x1, 0x1, 0xc422cb7790, 0x0, 0x0, 0x17ca1c0, 0x0, ...)
    /usr/local/go/src/crypto/x509/verify.go:304 +0x54
crypto/x509.(*Certificate).Verify(0xc42068a480, 0xc422ebe010, 0x9, 0xc42233c660, 0xc422322db0, 0xecffe9bf9, 0x70bf81a, 0x2a26ae0, 0x0, 0x0, ...)
    /usr/local/go/src/crypto/x509/verify.go:265 +0x1cd
crypto/tls.(*clientHandshakeState).doFullHandshake(0xc422cb7e20, 0xc421eaa0c0, 0x3a)
    /usr/local/go/src/crypto/tls/handshake_client.go:300 +0x221f
crypto/tls.(*Conn).clientHandshake(0xc420cac000, 0x1b8c1b8, 0xc420cac108)
    /usr/local/go/src/crypto/tls/handshake_client.go:228 +0xfd1
crypto/tls.(*Conn).Handshake(0xc420cac000, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1260 +0x1b8
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials.(*tlsCreds).ClientHandshake.func1(0xc422cf4a80, 0xc420cac000)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials/credentials.go:158 +0x2b
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials.(*tlsCreds).ClientHandshake
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials/credentials.go:159 +0x123

goroutine 19044 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe0000000d)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68580)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19048 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3432 +0x334
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc42228f8a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19046 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe0000000f)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f685c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19038 [semacquire]:
sync.runtime_notifyListWait(0xc421d11250, 0xe00000008)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc421d11240)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420d5d550, 0xc420370480)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68380)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19076 [select]:
github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:90 +0x16f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc42228fd00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19077 [runnable]:
github.com/cockroachdb/cockroach/vendor/github.com/codahale/hdrhistogram.(*iterator).next(0xc42012c760, 0x1)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/codahale/hdrhistogram/hdr.go:465
github.com/cockroachdb/cockroach/vendor/github.com/codahale/hdrhistogram.(*Histogram).ValueAtQuantile(0xc422171200, 0x4058f9999999999a, 0x40af560000000000)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/codahale/hdrhistogram/hdr.go:251 +0xb6
github.com/cockroachdb/cockroach/pkg/server/status.eachRecordableValue.func1(0xc421fd7e60, 0x14, 0x191c580, 0xc4223c5f20)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/status/recorder.go:377 +0xc4
github.com/cockroachdb/cockroach/pkg/util/metric.(*Registry).Each.func1(0x191c580, 0xc4223c5f20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/metric/registry.go:121 +0x6c
github.com/cockroachdb/cockroach/pkg/util/metric.(*Histogram).Inspect(0xc4223c5f20, 0xc420cc0e60)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/metric/metric.go:247 +0x81
github.com/cockroachdb/cockroach/pkg/util/metric.(*Registry).Each(0xc4223b1880, 0xc4220bf6b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/metric/registry.go:122 +0x125
github.com/cockroachdb/cockroach/pkg/server/status.eachRecordableValue(0xc4223b1880, 0xc4220bf6a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/status/recorder.go:387 +0x6f
github.com/cockroachdb/cockroach/pkg/server/status.(*MetricsRecorder).GetStatusSummary(0xc420adb500, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/status/recorder.go:268 +0x40e
github.com/cockroachdb/cockroach/pkg/server/status.(*MetricsRecorder).WriteStatusSummary(0xc420adb500, 0x2ae3593c3b80, 0xc421d19230, 0xc42233fa40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/status/recorder.go:303 +0xb9
github.com/cockroachdb/cockroach/pkg/server.(*Node).writeSummaries.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:711 +0x5c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc420370480, 0xc42012ce20, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:224 +0x100
github.com/cockroachdb/cockroach/pkg/server.(*Node).writeSummaries(0xc420c2f080, 0x2ae3593c3b80, 0xc421d19230, 0xc421686320, 0x1)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:712 +0x7d
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteSummaries.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:688 +0x79
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421d19260)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19078 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:751 +0x12b3
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68ae0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19079 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*Executor).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:332 +0x15f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421f68b60)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19080 [select]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:101 +0x314
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc42205bc00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19081 [IO wait]:
net.runtime_pollWait(0x2ae359409318, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42074f170, 0x72, 0xc420bbae08, 0xc420014220)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42074f170, 0x2552c80, 0xc420014220)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc42074f110, 0x0, 0x254fa00, 0xc421da8360)
    /usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc421840190, 0x62025e, 0xc420bbaeb8, 0xd33a1b)
    /usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc421840190, 0x1b8a1e8, 0xc4223b1e40, 0x256a8a0, 0xc4212da1d8)
    /usr/local/go/src/net/tcpsock.go:222 +0x49
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc4223b1e40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0xa4
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:625 +0x2f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421609a60)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 19082 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:194 +0x3db
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370480, 0xc421ebac30)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

ERROR: exit status 255

203 runs completed, 1 failures, over 13m3s
make: *** [stress] Error 1
Makefile:147: recipe for target 'stress' failed

cockroachdb / cockroach

github.com/cockroachdb/cockroach/pkg/kv: TestMultiRangeBoundedBatchDelRangeBoundary failed under stress #12665