search

cockroachdb / cockroach

CockroachDB — the cloud native, distributed SQL database designed for high availability, effortless scale, and control over data placement.
https://www.cockroachlabs.com
Other
30.1k stars 3.81k forks source link

teamcity: failed tests on master: test (proposer evaluated kv)/TestRetryableErrorForWrongTxn #13461

Closed cockroach-teamcity closed 7 years ago

cockroach-teamcity commented 7 years ago

The following tests appear to have failed:

#142620:

--- FAIL: test (proposer evaluated kv)/TestRetryableErrorForWrongTxn (0.049s)
Test ended in panic.
panic: test timed out after 2m0s

goroutine 229741 [running]:
panic(0x1854c60, 0xc421119100)
    /usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
    /usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
    /usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive]:
testing.(*T).Run(0xc4200923c0, 0x1a5f87e, 0x1d, 0x1c57450, 0xc4200d5c01)
    /usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc4200923c0)
    /usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc4200923c0, 0xc420950e00)
    /usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x1c58ee8, 0x2627d80, 0x79, 0x79, 0x2b51ff8)
    /usr/local/go/src/testing/testing.go:799 +0x319
testing.(*M).Run(0xc420950ef8, 0x2b51b20)
    /usr/local/go/src/testing/testing.go:743 +0x85
github.com/cockroachdb/cockroach/pkg/sql_test.TestMain(0xc420950ef8)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/main_test.go:227 +0xbd
main.main()
    github.com/cockroachdb/cockroach/pkg/sql/_test/_testmain.go:642 +0xc6

goroutine 17 [syscall, 1 minutes, locked to thread]:
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 20 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).flushDaemon(0x2b16280)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1012 +0x77
created by github.com/cockroachdb/cockroach/pkg/util/log.init.1
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:583 +0xc4

goroutine 21 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0x2b16280)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1039 +0x5b
created by github.com/cockroachdb/cockroach/pkg/util/log.init.1
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:584 +0xe8

goroutine 229871 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x39e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421192200)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229973 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*Executor).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:342 +0x165
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422b18240)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229886 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:478 +0x71
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4204b0060)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229928 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc400000018)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10c60)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229954 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x39e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421192280)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229955 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x39e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4211922a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229807 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:639 +0x4f0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422b18100)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229889 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4214369a0, 0xc42002e1c8, 0xc420582090, 0x18a4c00, 0x2611b50)
    <autogenerated>:9 +0x7f
crypto/tls.(*listener).Accept(0xc421436a00, 0xc420582060, 0x18a4c00, 0x2611b50, 0x1945280)
    /usr/local/go/src/crypto/tls/tls.go:52 +0x37
net/http.(*Server).Serve(0xc420c32680, 0x263ad20, 0xc421436a00, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:2273 +0x1ce
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:501 +0x42
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421436a20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229974 [select]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:101 +0x314
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc420c0af90)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229896 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func1(0xc4212d6d70)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:382 +0x44
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:383 +0x702

goroutine 229971 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteSummaries.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:717 +0x26d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc420580270)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229939 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startCoalescedHeartbeatsLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3441 +0x167
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc420585420)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229890 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:505 +0x88
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e372c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229956 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x39e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4211922e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229880 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4213aad00, 0x2b5e7d9867d0, 0xc420b08300, 0x1dcd6500, 0x0, 0xc4213aad48, 0x2, 0xc422a0a320, 0x1, 0x1, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1153 +0x157c
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4213aad00, 0x2b5e7d9867d0, 0xc420b08300, 0x2, 0xc422a0a320, 0x1, 0x1, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:405 +0x31a
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4213aad00, 0x2b5e7da01800, 0xc42042b0e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42080d5c0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:474 +0x16e
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4213aad00, 0x2b5e7da01800, 0xc42042b0e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42080d5c0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:926 +0x2c7
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4213aad00, 0x2b5e7da01800, 0xc42042b0e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42080d5c0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:805 +0x41f
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4213aad00, 0x2b5e7da01800, 0xc42042b0e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42080d500, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:620 +0x2ed
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421b6f290, 0x2b5e7da01800, 0xc42042b0e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc422a7d388, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:422 +0x7e9
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendInternal(0xc422a7d340, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc422a7d388, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:591 +0xfd
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).send(0xc422a7d340, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:737 +0x50d
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:321 +0x58
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0xc420719470, 0xc421ba0000, 0x0, 0xc420919200)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:426 +0x168
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc422a7d340, 0xc421ba0000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:321 +0xc6
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Get(0xc422a7d340, 0x19731e0, 0xc422a0a180, 0xc422a0a180, 0x18, 0x18, 0xc4222f2601, 0xc422a0a180, 0xc420719558)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:176 +0x99
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).GetProto(0xc422a7d340, 0x19731e0, 0xc422a0a180, 0x263f6a0, 0xc4222f2740, 0x10, 0xc420919050)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:184 +0x49
github.com/cockroachdb/cockroach/pkg/sql.getDescriptor(0xc422a7d340, 0x2639ae0, 0xc422223be0, 0x264d840, 0xc420796fc0, 0xc420ed2160, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:212 +0x203
github.com/cockroachdb/cockroach/pkg/sql.getTableOrViewDesc(0xc422a7d340, 0x2637160, 0xc4209cfb48, 0xc42042b1d0, 0xc4c0000000, 0x0, 0xc420719800)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/table.go:152 +0x23d
github.com/cockroachdb/cockroach/pkg/sql.getTableDesc(0xc422a7d340, 0x2637160, 0xc4209cfb48, 0xc42042b1d0, 0x0, 0x0, 0xc42042b1d0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/table.go:170 +0x4d
github.com/cockroachdb/cockroach/pkg/sql.(*planner).getTableDesc(0xc4209cfc60, 0xc42042b1d0, 0x0, 0xc42042b1d0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/table.go:163 +0x53
github.com/cockroachdb/cockroach/pkg/sql.(*planner).AlterTable(0xc4209cfc60, 0xc42042b0b0, 0x2b5e7eb94898, 0xc42042b0b0, 0xc422bc6180, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/alter_table.go:45 +0x7f
github.com/cockroachdb/cockroach/pkg/sql.(*planner).newPlan(0xc4209cfc60, 0x2b5e7eb94898, 0xc42042b0b0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x47, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:268 +0x49d
github.com/cockroachdb/cockroach/pkg/sql.(*planner).makePlan(0xc4209cfc60, 0x2b5e7eb94898, 0xc42042b0b0, 0xc40f09be01, 0x2b14c40, 0xc4207199b8, 0xc69122, 0xed02c0d9c)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:195 +0x81
github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execStmt(0xc422bc6180, 0x2b5e7eb94898, 0xc42042b0b0, 0xc4209cfc60, 0xc420710001, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:1342 +0xa3
github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execStmtInOpenTxn(0xc422bc6180, 0x2b5e7eb94898, 0xc42042b0b0, 0xc4209cfc60, 0x101, 0xc4209cfb68, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:1143 +0x2a5
github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execStmtsInCurrentTxn(0xc422bc6180, 0xc4222b8770, 0x1, 0x1, 0xc4209cfc60, 0xc4209cfb68, 0x101, 0x6, 0xc4222b87a0, 0x33, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:881 +0x788
github.com/cockroachdb/cockroach/pkg/sql.runTxnAttempt(0xc422bc6180, 0xc4209cfc60, 0x1, 0xc4209cfb68, 0xc4222b87b0, 0xc4222b8770, 0x1, 0x1, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:797 +0xf8
github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execRequest.func2(0xc422a7d340, 0xc4222b87b0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:636 +0x2b7
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Exec(0xc422a7d340, 0xc420a50101, 0xc422e36080, 0xc420a5cc40, 0xc400000000, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:513 +0x1c4
github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execRequest(0xc422bc6180, 0xc4209cfb00, 0x1a9c141, 0x48, 0x0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:658 +0x5e3
github.com/cockroachdb/cockroach/pkg/sql.(*Executor).ExecuteStatements(0xc422bc6180, 0xc4209cfb00, 0x1a9c141, 0x48, 0x0, 0x0, 0x0, 0x0, 0x4000000000000000)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:471 +0x11f
github.com/cockroachdb/cockroach/pkg/migrations.eventlogUniqueIDDefault(0x2b5e7da01800, 0xc4215b6ea0, 0x2636ca0, 0xc4220ad4f0, 0xc422bc6180, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/migrations/migrations.go:260 +0x1be
github.com/cockroachdb/cockroach/pkg/migrations.(*Manager).EnsureMigrations(0xc4225998c0, 0x2b5e7da01800, 0xc4215b6ea0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/migrations/migrations.go:201 +0x960
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start(0xc42154f680, 0x2b5e7da01800, 0xc4215b6ea0, 0x2b5e7d9b3080, 0xc420012df0)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:686 +0x15bf
github.com/cockroachdb/cockroach/pkg/server.(*TestServer).Start(0xc420d500a0, 0x26310e0, 0xc420c32000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/testserver.go:291 +0x184
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerRaw(0x26310e0, 0xc420c32000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:164 +0x11a
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServer(0x264ea40, 0xc4200920c0, 0x26310e0, 0xc420c32000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:134 +0x55
github.com/cockroachdb/cockroach/pkg/sql_test.TestRetryableErrorForWrongTxn(0xc4200920c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/txn_restart_test.go:1232 +0xeb
testing.tRunner(0xc4200920c0, 0x1c57450)
    /usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
    /usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 229894 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1037 +0x4de
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421809630)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229872 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x39e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421192240)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229924 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc400000014)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10bc0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229473 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc400000011)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10b40)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229944 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1349 +0x542
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc42219d770)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229809 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).startSampleEnvironment.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:826 +0x16b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422b181a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229870 [runnable]:
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.MarshalColumnValue(0xc421b7abf9, 0x7, 0x2, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/table.go:1266 +0x337
github.com/cockroachdb/cockroach/pkg/sql.(*RowInserter).InsertRow(0xc4202ab440, 0x2b5e7da01800, 0xc4223efd40, 0x2636c60, 0xc422cf4000, 0xc4220ec070, 0x7, 0x7, 0x0, 0xc421b7ac30, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/rowwriter.go:227 +0x115
github.com/cockroachdb/cockroach/pkg/sql.(*tableInserter).row(0xc4202ab440, 0x2b5e7da01800, 0xc4223efd40, 0xc4220ec070, 0x7, 0x7, 0xc, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:94 +0x89
github.com/cockroachdb/cockroach/pkg/sql.(*insertNode).Next(0xc420eda580, 0x264cb00, 0xc420eda580, 0xc421c22dd0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/insert.go:312 +0x2a3
github.com/cockroachdb/cockroach/pkg/sql.countRowsAffected(0x264cb00, 0xc420eda580, 0xc420eda580, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:768 +0x88
github.com/cockroachdb/cockroach/pkg/sql.(*planner).exec(0xc422098000, 0x1aa2868, 0x81, 0xc420d2d560, 0x6, 0x6, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/planner.go:385 +0x151
github.com/cockroachdb/cockroach/pkg/sql.InternalExecutor.ExecuteStatementInTransaction(0xc421ada460, 0x1a49f2a, 0xf, 0xc420eda000, 0x1aa2868, 0x81, 0xc420d2d560, 0x6, 0x6, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:47 +0x12b
github.com/cockroachdb/cockroach/pkg/storage.(*Store).insertRangeLogEvent(0xc422840000, 0xc420eda000, 0xed02c0d9c, 0xf35a9e1, 0x2b14c40, 0x2, 0x1, 0x1a3c8de, 0x5, 0xc420d2d500, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/log.go:108 +0x359
github.com/cockroachdb/cockroach/pkg/storage.(*Store).logSplit(0xc422840000, 0xc420eda000, 0x2, 0xc4223b7d58, 0x1, 0x8, 0xc422400098, 0x1, 0x8, 0xc4223b7d80, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/log.go:143 +0x329
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).adminSplitWithDescriptor.func1(0xc420eda000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:2426 +0x314
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Txn.func1(0xc420eda000, 0xc422400300, 0x4000000000000000, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:476 +0x2f
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Exec(0xc420eda000, 0xc422400101, 0x0, 0xc4224002f0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:513 +0x1c4
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Txn(0xc4220ad4f0, 0x2b5e7eb94800, 0xc420d2c300, 0xc422596640, 0xc421c238e0, 0x2)
    /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:477 +0x204
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).adminSplitWithDescriptor(0xc420a71800, 0x2b5e7eb94800, 0xc420d2c300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc422400098, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:2458 +0xab4
github.com/cockroachdb/cockroach/pkg/storage.(*splitQueue).process(0xc420584ba0, 0x2b5e7eb94800, 0xc420d2c300, 0xc420a71800, 0xc42293d800, 0x13, 0x20, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/split_queue.go:105 +0x262
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processReplica(0xc420e76600, 0x2b5e7da01800, 0xc4223ee030, 0xc420a71800, 0xc422e36080, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:603 +0x3ad
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1.2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:514 +0xfd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc42088c000, 0xc421c23e30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:239 +0x105
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:523 +0x368
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4211921c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229935 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc40000000f)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10d40)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229930 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc40000001a)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10ca0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229941 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func3()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1220 +0x165
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10de0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229982 [select]:
github.com/cockroachdb/cockroach/pkg/migrations.(*Manager).EnsureMigrations.func2(0x2b5e7da01800, 0xc4215b6ea0)
    /go/src/github.com/cockroachdb/cockroach/pkg/migrations/migrations.go:162 +0x2da
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc42088c000, 0x22fd4b6, 0x18, 0xb0, 0x0, 0x0, 0xc420cedae0, 0x2b5e7da01800, 0xc4215b6ea0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:283 +0xdf
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:284 +0x14a

goroutine 229942 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1349 +0x542
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc42219d680)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229849 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc421b6f290, 0x2b5e7da01800, 0xc420cd7d70, 0x39417463cb09ef23, 0xdf49616e207f8995)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:651 +0x41f
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func2(0x2b5e7da01800, 0xc4223efce0)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:923 +0x51
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc42088c000, 0x2308714, 0x16, 0x39c, 0x264e680, 0xc42155efa0, 0xc422591540, 0x2b5e7da01800, 0xc4223efce0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:283 +0xdf
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:284 +0x14a

goroutine 229990 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).wait(0xc4216022c0, 0x2b5e7eb94800, 0xc420d2c120, 0x100, 0x0, 0x15, 0x1c5007f, 0x1c5007e)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:908 +0x183
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).getTransport(0xc421290a00, 0x2b5e7eb94800, 0xc420d2c120, 0x2b5e7eb94800, 0xc420d2c120, 0xc4217a4000, 0x0, 0x0, 0x1c50091)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:622 +0x2b4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x2b5e7eb94800, 0xc420d2c120, 0x1a5f2f1, 0x1d, 0x19953c0, 0xc422596040, 0x1982e20, 0xc422590080, 0xc421290a00, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:226 +0x3b0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x2b5e7eb94800, 0xc420d2c120, 0x1a5f2f1, 0x1d, 0x19953c0, 0xc422596040, 0x1982e20, 0xc422590080, 0xc421290a00, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:143 +0x19c
github.com/cockroachdb/cockroach/pkg/rpc.(*heartbeatClient).Ping(0xc4204720b8, 0x2b5e7eb94800, 0xc420d2c120, 0xc422596040, 0x0, 0x0, 0x0, 0xc422e36090, 0xc422e36090, 0xc4210fed30)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/heartbeat.pb.go:121 +0xd2
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).heartbeat(0xc420c32580, 0x262e320, 0xc4204720b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4218094f0, 0xf, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:343 +0x15d
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420c32580, 0xc421290a00, 0xc420585910, 0xf, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:299 +0x29a
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421e4a060)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229925 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc400000015)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10be0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229927 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc400000017)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10c40)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229989 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials.(*tlsCreds).ClientHandshake(0xc42002e348, 0x2b5e7eb94800, 0xc420d2a060, 0xc420585910, 0xf, 0x264d520, 0xc420b74008, 0x0, 0x0, 0x2b5e7d910c20, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials/credentials.go:160 +0x329
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client(0x2b5e7eb94800, 0xc420d2a060, 0xc420585910, 0xf, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:171 +0x1876
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.NewClientTransport(0x2b5e7eb94800, 0xc420d2a060, 0xc420585910, 0xf, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0xae
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc4216022c0, 0x688000, 0xc42088c000, 0x23089e3)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:759 +0x282
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4216022c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:558 +0x34
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:567 +0x3c7

goroutine 229936 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc400000019)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10d60)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229934 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc40000000e)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10d20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229737 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*idAllocator).start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/id_alloc.go:133 +0x7f2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421118260)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229966 [IO wait]:
net.runtime_pollWait(0x2b5e7f7b8a38, 0x72, 0x8)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4208cc840, 0x72, 0xc421a7b388, 0xc4200120c8)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4208cc840, 0x2635de0, 0xc4200120c8)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4208cc7e0, 0xc422702800, 0x400, 0x400, 0x0, 0x2635de0, 0xc4200120c8)
    /usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420472188, 0xc422702800, 0x400, 0x400, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:173 +0x70
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc422086840, 0xc422702800, 0x400, 0x400, 0x1997500, 0x0, 0x2b5e7da40fe8)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x113
crypto/tls.(*block).readFromUntil(0xc422335e60, 0x2b5e7da40fe8, 0xc422086840, 0x5, 0xc422086840, 0x17)
    /usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc4223f8380, 0x516, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).readHandshake(0xc4223f8380, 0x539, 0x0, 0x0, 0x4)
    /usr/local/go/src/crypto/tls/conn.go:929 +0x8f
crypto/tls.(*serverHandshakeState).doFullHandshake(0xc421a7bb08, 0xc422798b00, 0x0)
    /usr/local/go/src/crypto/tls/handshake_server.go:447 +0x5a2
crypto/tls.(*Conn).serverHandshake(0xc4223f8380, 0x1c59888, 0xc4223f8488)
    /usr/local/go/src/crypto/tls/handshake_server.go:86 +0x316
crypto/tls.(*Conn).Handshake(0xc4223f8380, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1262 +0x2c3
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials.(*tlsCreds).ServerHandshake(0xc42002e058, 0x264d400, 0xc422086840, 0x1, 0x1, 0x26426e0, 0xc42002e1a8, 0xc4203273e0, 0xc42002e1b0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials/credentials.go:175 +0xd2
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).useTransportAuthenticator(0xc420092180, 0x264d400, 0xc422086840, 0x294d053b982a0e94, 0x2e120c4adc80b7b2, 0x294d053b982a0e94, 0x2e120c4adc80b7b2, 0xc420166f68, 0xc807ef)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:338 +0x50
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc420092180, 0x264d400, 0xc422086840)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:406 +0x5a
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:399 +0x5ac

goroutine 230012 [runnable]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).AnnotateCtx(0xc4212c1340, 0x2b5e7da01800, 0xc420919500, 0x100, 0x1a639fb)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:288
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4212c1340, 0x2b5e7da01800, 0xc420919500, 0xc42146bec8, 0x1f, 0x0, 0x14a115d132b86b72)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:836 +0x5e
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc4214321e0, 0xc42146bea0, 0x2b5e7da01800, 0xc420919500, 0xc806bf, 0x1, 0x1)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:193 +0x49c
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x264e680, 0xc42146bf40, 0xc4214321e0, 0xc42146bea0, 0x2b5e7da01800, 0xc420919500, 0xc42291f380)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:207 +0xbd
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:210 +0x142

goroutine 229893 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:366 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421436ac0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229972 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:725 +0x1305
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422b181c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229472 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:164 +0x55
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10b20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229884 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:98 +0x70
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421436740)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229892 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func8()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:525 +0x295
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4204b00f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229922 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc400000012)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10b80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229891 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4214368e0, 0x1c58be8, 0xc420092180, 0x264d400, 0xc422086840)
    <autogenerated>:9 +0x7f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc420092180, 0x2643060, 0xc4214368e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:369 +0x187
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:516 +0x43
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421436a40)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229957 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc4220b60c0, 0x2b5e7da01800, 0xc4215b6ea0, 0xed02c0d9c, 0xc40e72b7d7, 0x2b14c40, 0xc422e36080, 0xc42088c000, 0xc42170e000, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x254
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc42170e000, 0xc4205523f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:269 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc4205523f0, 0xc4217443c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:351 +0x303
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:271 +0x2ca
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421192300)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229937 [semacquire]:
sync.runtime_Semacquire(0xc4228ee204)
    /usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc4228ee1f8)
    /usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc4228ee160)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:180 +0x34
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3393 +0x2a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc420585400)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229887 [IO wait]:
net.runtime_pollWait(0x2b5e7f7b8738, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42047dcd0, 0x72, 0xc4211d1e08, 0xc4200120c8)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42047dcd0, 0x2635de0, 0xc4200120c8)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc42047dc70, 0x0, 0x2632f20, 0xc422b18020)
    /usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42002e1c8, 0xc4211d1ef0, 0xc4211d1f00, 0x67ec20)
    /usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42002e1c8, 0x1c57d48, 0xc422e37280, 0xc420e58030, 0x0)
    /usr/local/go/src/net/tcpsock.go:222 +0x49
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc422e37280, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0xa4
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func3()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:490 +0x2f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4214369c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229869 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x39e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421192180)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229938 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3410 +0x334
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc420585410)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229881 [chan receive]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:136 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4203ddc80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229895 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1075 +0x399
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421809640)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229931 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc40000001b)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10cc0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229885 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:98 +0x70
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421436780)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 230003 [runnable]:
math/big.nat.montgomery(0xc4202d5560, 0x10, 0x24, 0xc421bd1cc0, 0x10, 0x26, 0xc421bd1cc0, 0x10, 0x26, 0xc421b6e8f0, ...)
    /usr/local/go/src/math/big/nat.go:211
math/big.nat.expNNMontgomery(0xc4202d5560, 0x10, 0x26, 0xc4210e6000, 0x10, 0x25, 0xc4211f5540, 0x10, 0x25, 0xc421b6e8f0, ...)
    /usr/local/go/src/math/big/nat.go:1140 +0x79c
math/big.nat.expNN(0x0, 0x0, 0x0, 0xc4210e6000, 0x20, 0x45, 0xc4211f5540, 0x10, 0x25, 0xc421b6e8f0, ...)
    /usr/local/go/src/math/big/nat.go:955 +0x990
math/big.(*Int).Exp(0xc421687570, 0xc421687590, 0xc420ece2a0, 0xc420ece0e0, 0xc4225a2f80)
    /usr/local/go/src/math/big/int.go:420 +0xf2
crypto/rsa.decrypt(0x262d560, 0xc420103c50, 0xc421e4e600, 0xc421687590, 0x0, 0x0, 0x0)
    /usr/local/go/src/crypto/rsa/rsa.go:508 +0x447
crypto/rsa.decryptAndCheck(0x262d560, 0xc420103c50, 0xc421e4e600, 0xc4216876e0, 0xc4216876e0, 0x100, 0x0)
    /usr/local/go/src/crypto/rsa/rsa.go:542 +0x56
crypto/rsa.SignPKCS1v15(0x262d560, 0xc420103c50, 0xc421e4e600, 0x5, 0xc422a00c40, 0x20, 0x20, 0xc421687778, 0x6370ef, 0x18f8c20, ...)
    /usr/local/go/src/crypto/rsa/pkcs1v15.go:250 +0x2b1
crypto/rsa.(*PrivateKey).Sign(0xc421e4e600, 0x262d560, 0xc420103c50, 0xc422a00c40, 0x20, 0x20, 0x2633420, 0xc42173db00, 0xc420890000, 0x941, ...)
    /usr/local/go/src/crypto/rsa/rsa.go:102 +0x167
crypto/tls.(*clientHandshakeState).doFullHandshake(0xc421687e20, 0xc422345780, 0x3a)
    /usr/local/go/src/crypto/tls/handshake_client.go:511 +0xd7e
crypto/tls.(*Conn).clientHandshake(0xc420a84000, 0x1c59888, 0xc420a84108)
    /usr/local/go/src/crypto/tls/handshake_client.go:228 +0xfd1
crypto/tls.(*Conn).Handshake(0xc420a84000, 0x0, 0x0)
    /usr/local/go/src/crypto/tls/conn.go:1260 +0x1b8
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials.(*tlsCreds).ClientHandshake.func1(0xc420d2a360, 0xc420a84000)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials/credentials.go:158 +0x2b
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials.(*tlsCreds).ClientHandshake
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials/credentials.go:159 +0x123

goroutine 229873 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x39e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc421192260)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229806 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:680 +0x2bd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422b180e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229923 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc400000013)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10ba0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229926 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc400000016)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10c20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229976 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:200 +0x2a1
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc42092e370)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229883 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1204 +0xc76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4212a6c90)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229929 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc400000010)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10c80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229882 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).printStatsLoop(0xc421b6f290, 0x2b5e7da01800, 0xc4215b6ea0)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:209 +0xbdc
github.com/cockroachdb/cockroach/pkg/kv.NewTxnCoordSender.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:193 +0x98
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4218084f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229933 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc40000001d)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10d00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229975 [IO wait]:
net.runtime_pollWait(0x2b5e7f7b8c78, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42047dc60, 0x72, 0xc4210c8d98, 0xc4200120c8)
    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42047dc60, 0x2635de0, 0xc4200120c8)
    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc42047dc00, 0x0, 0x2632f20, 0xc4220b5100)
    /usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42002e1a8, 0x65e3ab, 0xc4210c8e48, 0x11b89bb)
    /usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42002e1a8, 0x1c57d50, 0xc422e37240, 0x264d520, 0xc420472188)
    /usr/local/go/src/net/tcpsock.go:222 +0x49
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc422e37240, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0xa4
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:637 +0x2f
sync.(*Once).Do(0xc421809590, 0xc4210c8f60)
    /usr/local/go/src/sync/once.go:44 +0xdb
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:638 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4229277e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229932 [semacquire]:
sync.runtime_notifyListWait(0xc4217da610, 0xc40000001c)
    /usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4217da600)
    /usr/local/go/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4228ee160, 0xc42088c000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:205 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc422e10ce0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229943 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4()
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1349 +0x542
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc42219d6d0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229888 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc421436980, 0xc420582000, 0x18a4c00, 0x2611b50, 0x1945280)
    <autogenerated>:9 +0x7f
net/http.(*Server).Serve(0xc420c32700, 0x2643060, 0xc421436980, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:2273 +0x1ce
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func4()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:494 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc4214369e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

goroutine 229970 [select]:
github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1()
    /go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:90 +0x16f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42088c000, 0xc420e583d0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:209 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x66

Please assign, take a look and update the issue accordingly.

a-robinson commented 7 years ago

13452