search

cockroachdb / cockroach

CockroachDB — the cloud native, distributed SQL database designed for high availability, effortless scale, and control over data placement.
https://www.cockroachlabs.com
Other
30.01k stars 3.79k forks source link

github.com/cockroachdb/cockroach/pkg/ccl/backupccl: TestBackupRestoreResume/restore failed under stress #26120

Closed cockroach-teamcity closed 6 years ago

cockroach-teamcity commented 6 years ago

SHA: https://github.com/cockroachdb/cockroach/commits/cbbc13cd30b14bc16cd8ebe56be093f264465ebe

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=680968&tab=buildLog


goroutine 45959 [running]:
github.com/cockroachdb/cockroach/pkg/util/log.getStacks(0x54a2c01, 0xc4204bc780, 0x0, 0x12)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:956 +0xdd
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).outputLogEntry(0x4adbf40, 0xc400000004, 0x453bced, 0x12, 0xf21, 0xc420e50e00, 0xd1)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:834 +0x7f4
github.com/cockroachdb/cockroach/pkg/util/log.addStructured(0x37be100, 0xc42410b770, 0xc400000004, 0x2, 0x321fdf1, 0x6, 0xc422307b88, 0x2, 0x2)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/structured.go:154 +0x31e
github.com/cockroachdb/cockroach/pkg/util/log.logDepth(0x37be100, 0xc42410b770, 0x1, 0x4, 0x321fdf1, 0x6, 0xc422307b88, 0x2, 0x2)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:55 +0x9a
github.com/cockroachdb/cockroach/pkg/util/log.Fatalf(0x37be100, 0xc42410b770, 0x321fdf1, 0x6, 0xc422307b88, 0x2, 0x2)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:149 +0x90
github.com/cockroachdb/cockroach/pkg/storage.fatalOnRaftReadyErr(0x37be100, 0xc42410b770, 0x323fac3, 0x16, 0x37985c0, 0xc4212e4a80)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3873 +0x219
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue.func1(0x37be100, 0xc42410b770, 0xc4227ec000, 0x37be100)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3576 +0x215
github.com/cockroachdb/cockroach/pkg/storage.(*Store).withReplicaForRequest(0xc424bd2480, 0x37be100, 0xc42410b770, 0xc423856ea0, 0xc422307e98, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3178 +0x169
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc424bd2480, 0x37be100, 0xc4237d0d50, 0xd)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3563 +0x302
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc4237d0d50)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:226 +0x2f2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc4237d0d50)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4244050c0, 0xc4277cc120, 0xc4244050b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1 [chan receive, 1 minutes]:
testing.(*T).Run(0xc420106960, 0x3240566, 0x17, 0x334c9b8, 0xc420607c01)
    /usr/local/go/src/testing/testing.go:825 +0x597
testing.runTests.func1(0xc420106960)
    /usr/local/go/src/testing/testing.go:1063 +0xa5
testing.tRunner(0xc420106960, 0xc420607d68)
    /usr/local/go/src/testing/testing.go:777 +0x16e
testing.runTests(0xc4203292a0, 0x4885b00, 0x2b, 0x2b, 0xbeba5e8a074d8cc0)
    /usr/local/go/src/testing/testing.go:1061 +0x4e2
testing.(*M).Run(0xc420480a00, 0x0)
    /usr/local/go/src/testing/testing.go:978 +0x2ce
github.com/cockroachdb/cockroach/pkg/ccl/backupccl.TestMain(0xc420480a00)
    /go/src/github.com/cockroachdb/cockroach/pkg/ccl/backupccl/main_test.go:31 +0x180
main.main()
    _testmain.go:136 +0x22b

goroutine 6 [syscall, 8 minutes]:
os/signal.signal_recv(0x90ab41)
    /usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:22 +0x30
created by os/signal.init.0
    /usr/local/go/src/os/signal/signal_unix.go:28 +0x4f

goroutine 8 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1158 +0x132
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x1b5

goroutine 9 [chan receive, 8 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:598 +0xf2
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:592 +0x1cd

goroutine 34 [select, 8 minutes, locked to thread]:
runtime.gopark(0x3352088, 0x0, 0x3221579, 0x6, 0x18, 0x1)
    /usr/local/go/src/runtime/proc.go:291 +0xf9
runtime.selectgo(0xc420091f50, 0xc4200b82a0)
    /usr/local/go/src/runtime/select.go:392 +0x11d4
runtime.ensureSigM.func1()
    /usr/local/go/src/runtime/signal_unix.go:549 +0x19f
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2361 +0x1

goroutine 51 [select]:
github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.(*worker).start(0xc4203310c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:144 +0x18f
created by github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.init.0
    /go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:29 +0x9b

goroutine 45763 [chan receive]:
testing.(*T).Run(0xc423a364b0, 0x3223661, 0x7, 0xc4227e6960, 0x1)
    /usr/local/go/src/testing/testing.go:825 +0x597
github.com/cockroachdb/cockroach/pkg/ccl/backupccl_test.TestBackupRestoreResume(0xc423a364b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/ccl/backupccl/backup_test.go:777 +0x482
testing.tRunner(0xc423a364b0, 0x334c9b8)
    /usr/local/go/src/testing/testing.go:777 +0x16e
created by testing.(*T).Run
    /usr/local/go/src/testing/testing.go:824 +0x565

goroutine 45979 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xee99abf700000bc6)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc4237d0f60)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc4237d0f60)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4244053a0, 0xc4277cc120, 0xc424405390)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46825 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x7a5164b20000077c)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bbac90)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bbac90)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424963a20, 0xc4228fc630, 0xc424963a10)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 921 [chan receive, 7 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4201fe900)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46004 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*Server).Start.func1(0x37be100, 0xc4209b0450)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:296 +0x1af
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4213e6930, 0xc4277cc120, 0xc423b34b00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45954 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xefc5de4500000be6)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194cb10)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194cb10)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424405000, 0xc4277cc120, 0xc424404ff0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47171 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x3518fd5200000642)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42337e240)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42337e240)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422f8c5f0, 0xc421304f30, 0xc422f8c5e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47414 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc420cbf500, 0x60, 0x3109020)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:240 +0x155
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc420cbf500, 0x58, 0xc421a815e0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:251 +0x3d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc4220bfba0, 0x30ba3a0, 0xc42097c6f0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:525 +0xadb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc422585d80, 0x30ba3a0, 0xc42097c6f0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x67
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4211af540, 0xc42097c6c0, 0xc421a816a8, 0xa4de23)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:366 +0x83
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x37d4080, 0xc4211af540, 0xc421610ec0, 0xc4226e82d0, 0x37be100, 0xc42097c6c0, 0x8d627d, 0xc420dca2c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:491 +0x5a
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x37be100, 0xc42097c6c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:506 +0x7e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4211af580, 0xc4228fc630, 0xc42097c690)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46819 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0xdcf40d9800000770)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bbaa80)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bbaa80)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424963960, 0xc4228fc630, 0xc424963950)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46951 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37be100, 0xc42100e660)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828280, 0xc421304f30, 0xc426828260)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 113 [chan receive, 8 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4207db440)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 114 [chan receive, 8 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4207db560)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 47158 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x37be100, 0xc4207553b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:473 +0x295
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421670f70, 0xc421304f30, 0xc4204608c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 36340 [chan receive, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4234085a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 29513 [chan receive, 3 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc423b36120)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46806 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x46bf2f4000000754)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bba7b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bba7b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4249637c0, 0xc4228fc630, 0xc4249637b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45757 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc42230a000)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x1d4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x294

goroutine 45772 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc42053e000)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:869 +0x31d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:281 +0x1651

goroutine 46960 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4209b7680)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46664 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x37be100, 0xc422320870)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:700 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a88f80, 0xc4228fc630, 0xc421b9ec60)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46912 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420755da0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:109 +0x205
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:95 +0x4d

goroutine 45835 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x37be100, 0xc42194c2a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:156 +0x63
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404a60, 0xc4277cc120, 0xc42125f580)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45924 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xd6af227d00000bd1)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194c4e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194c4e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404c00, 0xc4277cc120, 0xc424404bf0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46733 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x37be100, 0xc424bf20c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1558 +0x499
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828ae0, 0xc4228fc630, 0xc420c4db80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47194 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x2fa1b01800000625)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141b920)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141b920)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5320, 0xc421304f30, 0xc4216e5310)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45841 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xd6af227d00000bc1)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194c420)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194c420)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404ba0, 0xc4277cc120, 0xc424404b90)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47193 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x77b113e400000623)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141b8c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141b8c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5300, 0xc421304f30, 0xc4216e52f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 29514 [chan receive, 3 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc423b36240)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46592 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37be100, 0xc42100e4b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4268280d0, 0xc421304f30, 0xc4268280c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47179 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x37be100, 0xc42337eff0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422f8c8c0, 0xc421304f30, 0xc422d265e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47196 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0xb3fed17a0000061f)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141b950)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141b950)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5370, 0xc421304f30, 0xc4216e5350)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46593 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37be100, 0xc42100e4e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4268280f0, 0xc421304f30, 0xc4268280e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45963 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0x4da56e1f00000bd4)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc4237d0de0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc4237d0de0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424405160, 0xc4277cc120, 0xc424405150)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47267 [select]:
database/sql.(*DB).connectionOpener(0xc421284820, 0x37be040, 0xc42061f200)
    /usr/local/go/src/database/sql/sql.go:935 +0x13f
created by database/sql.OpenDB
    /usr/local/go/src/database/sql/sql.go:634 +0x2bd

goroutine 45917 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc423f325b0, 0x37be100, 0xc4237d10b0, 0xc4277cc120, 0x294c102c, 0xed29afb87, 0x0, 0xc42484b500, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:212 +0x24e
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc42484b500, 0x58)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x13c
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc420f77320, 0xc421f84e80)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:319 +0x2e2
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x37be100, 0xc4237d10b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272 +0x392
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42034eab0, 0xc4277cc120, 0xc420ee6320)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46968 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4236ef160, 0x30, 0x30d7cc0, 0x100000000000001, 0xc42236f5f0)
    <autogenerated>:1 +0x80
crypto/tls.(*listener).Accept(0xc4236ef1a0, 0xc420050128, 0x2f087a0, 0x4864a00, 0x31dab80)
    /usr/local/go/src/crypto/tls/tls.go:52 +0x53
net/http.(*Server).Serve(0xc4203bcf70, 0x37ab140, 0xc4236ef1a0, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:2770 +0x1f6
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5(0x37be100, 0xc42236f590)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1108 +0x89
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828540, 0xc421304f30, 0xc4236ef1c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46730 [chan receive]:
github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).execInternal(0xc421518870, 0x37be100, 0xc42348dd10, 0x32265f1, 0x9, 0x0, 0xc42348b640, 0x32b3a12, 0x52, 0xc420910220, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:490 +0x980
github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).queryInternal(0xc421518870, 0x37be100, 0xc424bf2180, 0x32265f1, 0x9, 0x0, 0xc42348b640, 0x32b3a12, 0x52, 0xc420910220, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:240 +0x112
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).Query(0xc421518870, 0x37be100, 0xc424bf2180, 0x32265f1, 0x9, 0x0, 0x32b3a12, 0x52, 0xc420910220, 0x2, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:226 +0x1a0
github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).maybeAdoptJob(0xc4262a2630, 0x37be100, 0xc424bf2180, 0x37a0c40, 0xc423576200, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:485 +0x170
github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func2(0x37be100, 0xc424bf2180)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:272 +0x190
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828ab0, 0xc4228fc630, 0xc420958b10)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 52260 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*hookFnNode).Next(0xc420d9bc80, 0x37be100, 0xc42348cd50, 0xc420eaa498, 0xc420eaa428, 0x37b21c0, 0xc420d9bc80, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/planhook.go:146 +0x1a4
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).forEachRow(0xc420eaa000, 0x37be100, 0xc42348cd50, 0xc420eaa498, 0xc420eaa428, 0x37b21c0, 0xc420d9bc80, 0xc42099e870, 0x92abc5, 0x92e1cc)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:764 +0x7d
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc420eaa000, 0x37be100, 0xc42348cd50, 0xc420eaa428, 0x3, 0x7f8edf33ec18, 0xc423a79b80, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:709 +0x2ff
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc420eaa000, 0x37be100, 0xc42348cd50, 0x37bf0c0, 0xc4208ecf00, 0xc423409320, 0x7, 0x7, 0xc424309f20, 0x1a, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:657 +0xbaf
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc420eaa000, 0x37be100, 0xc42348cd50, 0x37bf0c0, 0xc4208ecf00, 0xc423409320, 0x7, 0x7, 0xc424309f20, 0x1a, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:398 +0x1090
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc420eaa000, 0x37be100, 0xc42348cd50, 0x37bf0c0, 0xc4208ecf00, 0xc423409320, 0x7, 0x7, 0xc424309f20, 0x1a, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:92 +0x46f
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc420eaa000, 0x37be040, 0xc420b81100, 0xc426829f10, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1108 +0x1ebd
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc420dd0400, 0x37be040, 0xc420b81100, 0xc421281039, 0x4, 0xc421281043, 0x4, 0x0, 0x0, 0x37a2e40, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:401 +0x1e1
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc420dd0400, 0x37be040, 0xc420b81100, 0xc42150c000, 0x5400, 0x15000, 0xc422197c80, 0xc426829f10, 0xc426829f00, 0xc424c99ff0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:266 +0x1aa
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:265 +0x147b

goroutine 51718 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420cbab00)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1373 +0x2ac
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc420cbab00)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:949 +0x247
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:940 +0x173

goroutine 45744 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x37be100, 0xc420f76030)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1112 +0xbe
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404000, 0xc4277cc120, 0xc42005c680)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46949 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37be100, 0xc42100e600)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828230, 0xc421304f30, 0xc426828220)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 23902 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42290f7a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 47416 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc424014500, 0xc42168b130, 0x5, 0x5, 0xc421c6bd60, 0x8e79c0, 0x299ab60)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x19c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc424014500, 0xc42168b130, 0x5, 0x5, 0x8e6c7c, 0xc421c6bd60, 0xc4221e3970)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:131 +0x82
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc42097c900, 0xc42168b130, 0x5, 0x5, 0x1019163, 0xc421c6bd60, 0x1fffe0000114ab)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0x71
io.ReadAtLeast(0x3798d60, 0xc42097c900, 0xc42168b130, 0x5, 0x5, 0x5, 0xc4221e3a48, 0x1043409, 0xc42053e800)
    /usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x3798d60, 0xc42097c900, 0xc42168b130, 0x5, 0x5, 0x0, 0x0, 0x0)
    /usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420cbf600, 0xc42168b130, 0x5, 0x5, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:378 +0x15d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42168b120, 0x7fffffff, 0xb, 0x0, 0x0, 0x0, 0x0, 0xe)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:452 +0x99
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42168b120, 0x7f8ee4339050, 0x54a2f90, 0xc420cbf600, 0x0, 0x0, 0x30ba2c0, 0xc42457a680, 0x7fffffff, 0xc42722b560, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:578 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc42042da20, 0x30ba2c0, 0xc42457a680, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:757 +0x1e9
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc4211af600, 0x37be100, 0xc42097ca80, 0xc421297d40)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:434 +0x83
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x37d40e0, 0xc4211af600, 0xc423a36b40, 0x37be100, 0xc42097ca80, 0x8d627d, 0xc420670c00)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x109
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x37be100, 0xc42097ca80)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:353 +0x75
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4211af620, 0xc4277cc120, 0xc42097ca20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46973 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc423879170)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:109 +0x205
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:95 +0x4d

goroutine 45929 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xee99abf700000bc8)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc4237d0990)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc4237d0990)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404cc0, 0xc4277cc120, 0xc424404ca0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46666 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x37be100, 0xc422320900)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112 +0x350
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a88fd0, 0xc4228fc630, 0xc421b9eca0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46731 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2(0x37be100, 0xc424bba780)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1420 +0x149
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828ac0, 0xc4228fc630, 0xc423a3c040)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46741 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x37be100, 0xc4237d0030)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424962000, 0xc4228fc630, 0xc422712000)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45820 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x37be100, 0xc4281f6090)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112 +0x350
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4213e6830, 0xc4277cc120, 0xc423b34a40)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 41199 [chan receive, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc421132b40)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46715 [select]:
database/sql.(*DB).connectionOpener(0xc4248903c0, 0x37be040, 0xc420fa8640)
    /usr/local/go/src/database/sql/sql.go:935 +0x13f
created by database/sql.OpenDB
    /usr/local/go/src/database/sql/sql.go:634 +0x2bd

goroutine 46936 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x37be100, 0xc421ae4c00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x7d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4249625a0, 0xc421304f30, 0xc4237f71a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46646 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x37be100, 0xc421852210)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1112 +0xbe
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5690, 0xc4228fc630, 0xc42143eb80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46754 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x46bf2f4000000789)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc423b6fbf0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc423b6fbf0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4249630f0, 0xc4228fc630, 0xc4249630e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45914 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x37be100, 0xc4237d11d0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42034ea70, 0xc4277cc120, 0xc420ee62c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 51729 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc4211b0ac0, 0x1, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc4206de180, 0xc421543a00, 0xc4211b0ac0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4205cfb00)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:298 +0x101
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:296 +0x14cf

goroutine 46804 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x5b46c5bd00000755)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bba690)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bba690)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424963780, 0xc4228fc630, 0xc424963770)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46929 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x37be100, 0xc42080b980)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:738 +0x174
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e4d70, 0xc421304f30, 0xc420387920)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45976 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xd6af227d00000bfb)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194cd20)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194cd20)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424405340, 0xc4277cc120, 0xc424405330)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47248 [select]:
github.com/cockroachdb/cockroach/pkg/storage/compactor.(*Compactor).Start.func1(0x37be100, 0xc423ba8900)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/compactor/compactor.go:108 +0x23f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5b00, 0xc421304f30, 0xc423ba87e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46958 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37be100, 0xc42100e7b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828380, 0xc421304f30, 0xc426828370)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46497 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x37be100, 0xc4253cdad0)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1161 +0x5a5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a88480, 0xc4228fc630, 0xc423a88470)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47207 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x2bc171820000062f)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141bbc0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141bbc0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e55b0, 0xc421304f30, 0xc4216e5550)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 54870 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc421b5c4b0, 0xc42457ae30, 0x5, 0x5, 0xc420c6d220, 0x8e79c0, 0x299ab60)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x19c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421b5c4b0, 0xc42457ae30, 0x5, 0x5, 0x8e6c7c, 0xc420c6d220, 0xc4237828f0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:131 +0x82
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc423381ec0, 0xc42457ae30, 0x5, 0x5, 0x1019163, 0xc420c6d220, 0x1fffe00000002d)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0x71
io.ReadAtLeast(0x3798d60, 0xc423381ec0, 0xc42457ae30, 0x5, 0x5, 0x5, 0xc4237829c8, 0x1041979, 0xc4207f6000)
    /usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x3798d60, 0xc423381ec0, 0xc42457ae30, 0x5, 0x5, 0xc42348a398, 0xc4209108e0, 0xc42348a380)
    /usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42417d400, 0xc42457ae30, 0x5, 0x5, 0xc42365cc00, 0x16, 0x600)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:378 +0x15d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42457ae20, 0x7fffffff, 0xb, 0x0, 0x7210, 0xc420041220, 0x0, 0xe)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:452 +0x99
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42457ae20, 0x7f8ee4339050, 0x54a2f90, 0xc42417d400, 0x0, 0x0, 0x30f5ce0, 0xc420910920, 0x7fffffff, 0xc4207de660, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:578 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc421846c30, 0x30f5ce0, 0xc420910920, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:539 +0x27c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4242b4900, 0x30f5ce0, 0xc420910920, 0x3292630, 0x35)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x67
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*distSQLFlowStreamClient).Recv(0xc4203173c0, 0xc4210e8500, 0x3292630, 0x35)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/api.pb.go:331 +0x83
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*outbox).listenForDrainSignalFromConsumer.func1(0x37be040, 0xc4210e8500)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/outbox.go:328 +0xfb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4277cc120, 0x37be040, 0xc4210e8500, 0xc4225a80d1, 0xd, 0x37e5320, 0xc42262bcc0, 0xc42457ae80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xf4
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:259 +0x14d

goroutine 47050 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x111f8ccf00000648)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc421b35200)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc421b35200)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422f8c4d0, 0xc421304f30, 0xc422f8c4a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 12967 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42290ea20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
    /go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46331 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc4224907c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x1d4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x294

goroutine 46814 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x46bf2f4000000769)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bba9c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bba9c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4249638c0, 0xc4228fc630, 0xc4249638b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47522 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc421930000, 0xc423adf000, 0x37be100, 0xc420bac7b0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:321 +0x120
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0x1200

goroutine 46772 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x46bf2f400000078d)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bba390)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bba390)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424963430, 0xc4228fc630, 0xc424963420)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47432 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc4209b8240, 0x1, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc4214d43c0, 0xc4216e5be0, 0xc4209b8240)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4207f6000)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:298 +0x101
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:296 +0x14cf

goroutine 45958 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0x4da56e1f00000bdb)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194cb70)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194cb70)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4244050a0, 0xc4277cc120, 0xc424405080)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47146 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x77b113e40000064b)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141b5f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141b5f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5070, 0xc421304f30, 0xc4216e5060)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45944 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0x2a20103200000be3)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194c960)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194c960)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404ea0, 0xc4277cc120, 0xc424404e90)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47152 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x2fa1b01800000652)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141b7a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141b7a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5140, 0xc421304f30, 0xc4216e5130)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46736 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startLeaseRenewer.func1(0x37be100, 0xc424bf2120)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1607 +0x4b2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828b70, 0xc4228fc630, 0xc426828b60)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46783 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x9bdc470e0000075b)
    /usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
    /usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bba570)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bba570)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424963640, 0xc4228fc630, 0xc4249635d0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 54890 [runnable]:
cockroach-teamcity commented 6 years ago

SHA: https://github.com/cockroachdb/cockroach/commits/915a266778aab15c98fc7e18639583fbe6532399

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=684084&tab=buildLog

cockroach-teamcity commented 6 years ago

SHA: https://github.com/cockroachdb/cockroach/commits/915a266778aab15c98fc7e18639583fbe6532399

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=684084&tab=buildLog

windchan7 commented 6 years ago

26259