search

cockroachdb / cockroach

CockroachDB — the cloud native, distributed SQL database designed for high availability, effortless scale, and control over data placement.
https://www.cockroachlabs.com
Other
30.17k stars 3.82k forks source link

storage: TestGossipHandlesReplacedNode failed under stress #15585

Closed cockroach-teamcity closed 7 years ago

cockroach-teamcity commented 7 years ago

SHA: https://github.com/cockroachdb/cockroach/commits/0d4a3a475f4f8622bde6bd34e131013606bd5617

Parameters:

TAGS=
GOFLAGS=

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=240022&tab=buildLog

    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc42216c9f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420368680, 0xc421b79720, 0xc420247320)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65865 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc40000927a)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc420c45d40)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203685b0, 0xc421b79720, 0xc4202470a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65845 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc40000925b)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc420c45a40)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420368400, 0xc421b79720, 0xc420246c60)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66347 [select, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420a03520)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x666
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420a03520)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x1d9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0x715

goroutine 65563 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7ff51dc768f8, 0xc4224020c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x364
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b81f0, 0xc421b79720, 0xc4202dc0e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65742 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc400009287)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc42082d290)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420368220, 0xc421b79720, 0xc420246580)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65791 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc421c6a120, 0xc420bafa80, 0xc4202260a0, 0x12, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:356 +0x801
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1(0x7ff51dc768f8, 0xc4216f0e70)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:297 +0x6b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420306d90, 0xc4217a92c0, 0xc4216f0780)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65565 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7ff51dc768f8, 0xc422402120)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x364
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b8210, 0xc421b79720, 0xc4202dc180)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66041 [IO wait]:
net.runtime_pollWait(0x7ff51dd87540, 0x72, 0x8)
    /usr/local/go/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc4200a25a8, 0x72, 0x26feac0, 0x26f4690)
    /usr/local/go/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc4200a25a8, 0xc421546000, 0x8000)
    /usr/local/go/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).Read(0xc4200a2540, 0xc421546000, 0x8000, 0x8000, 0x0, 0x26feac0, 0x26f4690)
    /usr/local/go/src/net/fd_unix.go:250 +0x1b7
net.(*conn).Read(0xc42000e070, 0xc421546000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:181 +0x70
bufio.(*Reader).Read(0xc4221c8b40, 0xc4201de498, 0x9, 0x9, 0x30, 0xc42128edb8, 0x26fb180)
    /usr/local/go/src/bufio/bufio.go:213 +0x312
io.ReadAtLeast(0x26f6340, 0xc4221c8b40, 0xc4201de498, 0x9, 0x9, 0x9, 0x37, 0x37, 0x0)
    /usr/local/go/src/io/io.go:307 +0xa9
io.ReadFull(0x26f6340, 0xc4221c8b40, 0xc4201de498, 0x9, 0x9, 0x9ba77069327b9601, 0x9b00000000000037, 0x0)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4201de498, 0x9, 0x9, 0x26f6340, 0xc4221c8b40, 0x0, 0x100000000, 0x100000037, 0xc4216cd8c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4201de460, 0x2700ec0, 0xc421e841e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc42128eb70, 0xc421e841e0, 0xc421e841e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc421cf8d80)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1043 +0xcd
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0xcc6

goroutine 65774 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*sqlStats).startResetWorker.func1(0x7ff51dc768f8, 0xc420f29590)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/app_stats.go:264 +0x183
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203281d0, 0xc421b79720, 0xc42018a7c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 93900 [IO wait]:
net.runtime_pollWait(0x7ff51dd87f00, 0x72, 0x16)
    /usr/local/go/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc42186c148, 0x72, 0x26feac0, 0x26f4690)
    /usr/local/go/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc42186c148, 0xc421f50000, 0x8000)
    /usr/local/go/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).Read(0xc42186c0e0, 0xc421f50000, 0x8000, 0x8000, 0x0, 0x26feac0, 0x26f4690)
    /usr/local/go/src/net/fd_unix.go:250 +0x1b7
net.(*conn).Read(0xc4203caf18, 0xc421f50000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:181 +0x70
bufio.(*Reader).Read(0xc421e952c0, 0xc4217017d8, 0x9, 0x9, 0xc4212d7740, 0xc4225df658, 0xc4212d7558)
    /usr/local/go/src/bufio/bufio.go:213 +0x312
io.ReadAtLeast(0x26f6340, 0xc421e952c0, 0xc4217017d8, 0x9, 0x9, 0x9, 0x699ad0, 0xc4215a11e0, 0x4)
    /usr/local/go/src/io/io.go:307 +0xa9
io.ReadFull(0x26f6340, 0xc421e952c0, 0xc4217017d8, 0x9, 0x9, 0x647feb, 0xc4225df680, 0xc420261770)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4217017d8, 0x9, 0x9, 0x26f6340, 0xc421e952c0, 0x0, 0xc400000000, 0xc4225df6d8, 0xc42038a2f9)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4217017a0, 0xc4212d7770, 0xc4225df560, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4218636b0, 0xc4212d77a4, 0xc4212d77a0, 0x0, 0xc4225df560)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc4216fb680)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1028 +0x47
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0xcc6

goroutine 93751 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func2(0xc421fc45d0, 0xc421e8f2c0, 0xc4222c8300, 0x7ff51dc768f8, 0xc420efd3e0, 0xc421fc4600, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1065 +0x4e1
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc4222c8300, 0x7ff51dc768f8, 0xc420efd3e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1115 +0x1eb
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeReadOnlyBatch(0xc4222c8300, 0x7ff51dc768f8, 0xc420efd3e0, 0x14bab827c5a1d8c6, 0x0, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2005 +0x82b
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4222c8300, 0x7ff51dc768f8, 0xc420efd3e0, 0x14bab827c5a1d8c6, 0x0, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1419 +0x482
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4205e7400, 0x7ff51dc768f8, 0xc420efd3b0, 0x14bab827c5a1d8c6, 0x0, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2552 +0x7ae
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc421404480, 0x7ff51dc768f8, 0xc420efd380, 0x0, 0x0, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x1cf
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x7ff51dc768f8, 0xc420efd380, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x191
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421b79720, 0x7ff51dc768f8, 0xc420efd350, 0xc421fc5828, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:272 +0x12f
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420596300, 0x7ff51dc768f8, 0xc420efd350, 0xc4226f73a8, 0xc420efd350, 0x1f, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x101
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420596300, 0x7ff51dc768f8, 0xc420efd350, 0xc4226f73a8, 0x1f, 0xc420efd200, 0xdc8ca6)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0x99
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc420ddfc20, 0xc4226f7380, 0x7ff51dc768f8, 0xc420efd2f0, 0x18, 0xcfa0d4, 0xdc5790)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:257 +0x49f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420ddfc20, 0xc4226f7380, 0x7ff51dc768f8, 0xc420efd2f0, 0x2719aa0, 0x2b84708, 0xc421e8f260)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:270 +0x7e
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:283 +0x14e

goroutine 66134 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7ff51dc768f8, 0xc4216290b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x364
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c02e0, 0xc4217a92c0, 0xc4201e46a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65918 [select, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc420bac480)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x65a
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x10c2

goroutine 66057 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc400008950)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc4202a9f80)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb000, 0xc4217a92c0, 0xc4201fc200)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66135 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7ff51dc768f8, 0xc4216290e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x364
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c02f0, 0xc4217a92c0, 0xc4201e4700)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65763 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7ff51dc768f8, 0xc420f29410)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x181
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420328070, 0xc421b79720, 0xc420328060)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66079 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc400008926)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc4216286c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb1d0, 0xc4217a92c0, 0xc4201fc6c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66588 [select, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4216f5560, 0x1b7a1a8, 0xc4210c5638, 0x27152e0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x19b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7ff51dc768f8, 0xc4217236e0, 0x0, 0x0, 0x270d780, 0x2b84708, 0x0, 0x0, 0x2702a00, 0x2b84708, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xaa
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7ff51dc768f8, 0xc4217236e0, 0x1ad352f, 0x21, 0x1a89f00, 0xc421019768, 0x19fe0e0, 0xc4210e8000, 0xc42105a300, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xa4b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7ff51dc768f8, 0xc4217236e0, 0x1ad352f, 0x21, 0x1a89f00, 0xc421019768, 0x19fe0e0, 0xc4210e8000, 0xc42105a300, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x19c
github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc420d24208, 0x7ff51dc768f8, 0xc4217236e0, 0xc421019768, 0x0, 0x0, 0x0, 0x1, 0x7ff51dc768f8, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1872 +0xd2
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc4216f60a0, 0xc421019740, 0x7ff51dc768f8, 0xc4217236e0, 0xc420397578, 0xc42165d778, 0xff0cc3)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:261 +0x17f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4216f60a0, 0xc421019740, 0x7ff51dc768f8, 0xc4217236e0, 0x2719aa0, 0x2b84708, 0xc420e842a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:270 +0x7e
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:283 +0x14e

goroutine 65850 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc400009265)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc420c45b30)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203684a0, 0xc421b79720, 0xc420246d80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65858 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc40000926e)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc420c45bf0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420368540, 0xc421b79720, 0xc420246f80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66058 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc400008951)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc421628000)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb020, 0xc4217a92c0, 0xc4201fc220)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65974 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7ff51dc768f8, 0xc420abf110)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x181
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420306260, 0xc4217a92c0, 0xc420306250)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65971 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7ff51dc768f8, 0xc420abecf0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x181
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203061d0, 0xc4217a92c0, 0xc4203061c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65816 [chan receive, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x7ff51dc768f8, 0xc4207860f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:550 +0x88
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420333ca0, 0xc4217a92c0, 0xc420b4f140)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66021 [select, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420327ea0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:742 +0x743
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:161 +0x896

goroutine 65990 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*sqlStats).startResetWorker.func1(0x7ff51dc768f8, 0xc42128f050)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/app_stats.go:264 +0x183
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420306520, 0xc4217a92c0, 0xc42029a300)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65840 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc40000925c)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc42216c870)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203683a0, 0xc421b79720, 0xc420246b40)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65834 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc400009257)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc420c459e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420368330, 0xc421b79720, 0xc420246940)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 93896 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc4216fac00)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x65a
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x10c2

goroutine 66097 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc400008938)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc421628a80)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb2f0, 0xc4217a92c0, 0xc4201fc980)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66601 [select, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4217290e0, 0x1b7a1a8, 0xc42171d638, 0x27152e0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x19b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7ff51dc768f8, 0xc421715e30, 0x0, 0x0, 0x270d780, 0x2b84708, 0x0, 0x0, 0x2702a00, 0x2b84708, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xaa
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7ff51dc768f8, 0xc421715e30, 0x1ad352f, 0x21, 0x1a89f00, 0xc4211a52e8, 0x19fe0e0, 0xc4210d2900, 0xc4209b0300, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xa4b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7ff51dc768f8, 0xc421715e30, 0x1ad352f, 0x21, 0x1a89f00, 0xc4211a52e8, 0x19fe0e0, 0xc4210d2900, 0xc4209b0300, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x19c
github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421f1e580, 0x7ff51dc768f8, 0xc421715e30, 0xc4211a52e8, 0x0, 0x0, 0x0, 0x1, 0x7ff51dc768f8, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1872 +0xd2
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc4210d45f0, 0xc4211a52c0, 0x7ff51dc768f8, 0xc421715e30, 0x0, 0x0, 0xc421442f80)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:261 +0x17f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4210d45f0, 0xc4211a52c0, 0x7ff51dc768f8, 0xc421715e30, 0x2719aa0, 0x2b84708, 0xc4210d28a0)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:270 +0x7e
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:283 +0x14e

goroutine 66613 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc4216fb380)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1162 +0x4ed
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:283 +0x1183

goroutine 65752 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7ff51dc768f8, 0xc420abe000)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x181
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42037be70, 0xc421b79720, 0xc42037be60)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65786 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x7ff51dc768f8, 0xc4216f0990)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:115 +0x3e1
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420306c20, 0xc4217a92c0, 0xc420306c10)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65711 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4206b8ea0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x666
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4206b8ea0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x1d9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0x715

goroutine 66136 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7ff51dc768f8, 0xc421629110)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x364
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0300, 0xc4217a92c0, 0xc4201e4720)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65776 [IO wait, 29 minutes]:
net.runtime_pollWait(0x7ff51dd87780, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc4207d2378, 0x72, 0x0, 0xc4203084a0)
    /usr/local/go/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc4207d2378, 0xffffffffffffffff, 0x0)
    /usr/local/go/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).accept(0xc4207d2310, 0x0, 0x26fbec0, 0xc4203084a0)
    /usr/local/go/src/net/fd_unix.go:430 +0x1e5
net.(*TCPListener).accept(0xc4204381f8, 0xc422171e48, 0xd9220c, 0x69a230)
    /usr/local/go/src/net/tcpsock_posix.go:136 +0x2e
net.(*TCPListener).Accept(0xc4204381f8, 0x1b793b8, 0xc4223dcf00, 0x27171a0, 0xc420438358)
    /usr/local/go/src/net/tcpsock.go:228 +0x49
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc4223dcf00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x95
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10.1()
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:743 +0x2f
sync.(*Once).Do(0xc420184cc0, 0xc4214f3760)
    /usr/local/go/src/sync/once.go:44 +0xbe
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10(0x7ff51dc768f8, 0xc420abe0f0)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:744 +0x64
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203697a0, 0xc421b79720, 0xc4202bf400)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66137 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc4204f58c0, 0x7ff51dc768f8, 0xc421628f30, 0xed09a26b0, 0x6db5243, 0x2b478a0, 0xc420b4ecc0, 0xc4217a92c0, 0xc420596900, 0x1)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:210 +0x241
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc420596900, 0xc420a98b10)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272 +0x90
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc420a98b10, 0xc421e89540)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:356 +0x2ff
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x7ff51dc768f8, 0xc421628f30)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x238
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0310, 0xc4217a92c0, 0xc4201e4780)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66055 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc40000894e)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc4202a9f20)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bafe0, 0xc4217a92c0, 0xc4201fc1c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66044 [select, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func3(0x27152e0, 0xc421cf8d80, 0xc4216cd8c0, 0xc420b946e0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:236 +0x40f
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:256 +0xc94

goroutine 65999 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc400008946)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc4202a9a10)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201baf60, 0xc4217a92c0, 0xc4201fc040)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65826 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc40000924c)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc420c45860)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420368270, 0xc421b79720, 0xc4202466e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65705 [chan receive, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x7ff51dc768f8, 0xc420282ea0)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:386 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420184e80, 0xc421b79720, 0xc4201e5d20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 93882 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc421f96f70, 0x7ff51dc768f8, 0xc4220277d0, 0xc421384300, 0x2715ee0, 0xc421eee070, 0xc421b79720, 0xc421eee030, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:325 +0x3e2
github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1(0x7ff51dc768f8, 0xc4220277d0)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:132 +0x410
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42271bfe0, 0xc421b79720, 0xc421188980)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66100 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc40000893b)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc421628b10)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb320, 0xc4217a92c0, 0xc4201fca00)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65979 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7ff51dc768f8, 0xc420abec90)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x181
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420306330, 0xc4217a92c0, 0xc420306320)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65976 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7ff51dc768f8, 0xc420abf170)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x181
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203062c0, 0xc4217a92c0, 0xc4203062b0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66086 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc40000892d)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc421628810)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb240, 0xc4217a92c0, 0xc4201fc7e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65699 [chan receive, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x7ff51dc768f8, 0xc420282bd0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x70
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420184c30, 0xc421b79720, 0xc4201e5960)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65825 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7ff51dc768f8, 0xc421628f60)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x364
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0260, 0xc4217a92c0, 0xc4201e45c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65741 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc400009283)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc42082d0e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420368210, 0xc421b79720, 0xc420246520)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65818 [chan receive, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc420226140, 0xc421435200, 0x0, 0x0, 0xc42022d800)
    <autogenerated>:9 +0x7a
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc4204380a0, 0x7ff51dc768f8, 0xc42216cd50, 0xc4217a92c0, 0x270d180, 0xc420226140, 0xc421435200, 0x651348, 0x1938a60)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:121 +0x50
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9(0x7ff51dc768f8, 0xc42216cd50)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:625 +0x1b4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420333cc0, 0xc4217a92c0, 0xc420b4f280)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65758 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7ff51dc768f8, 0xc420abe0c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x181
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42037bfd0, 0xc421b79720, 0xc42037bfc0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65762 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7ff51dc768f8, 0xc420f293e0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x181
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420328050, 0xc421b79720, 0xc420328040)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65844 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc400009251)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc42216c780)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203683f0, 0xc421b79720, 0xc420246c40)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66132 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7ff51dc768f8, 0xc421629050)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x364
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c02c0, 0xc4217a92c0, 0xc4201e4640)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65994 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc400008941)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc4202a9860)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201baf10, 0xc4217a92c0, 0xc420255f20)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66612 [select, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc4216fb380)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x65a
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x10c2

goroutine 66644 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4211cdd00, 0xc420352890, 0x5, 0x5, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x556
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc4216f58c0, 0xc420352890, 0x5, 0x5, 0x80000c420722730, 0x670884, 0xc420c30b70)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x5c
io.ReadAtLeast(0x26fb040, 0xc4216f58c0, 0xc420352890, 0x5, 0x5, 0x5, 0xc421199d40, 0xc4210e7180, 0xc420c30c90)
    /usr/local/go/src/io/io.go:307 +0xa9
io.ReadFull(0x26fb040, 0xc4216f58c0, 0xc420352890, 0x5, 0x5, 0x0, 0x7, 0x1)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc420352880, 0x7fffffff, 0xc420c30ce8, 0xdc10d8, 0xc421bdc000, 0xc420c30cd8, 0x1c, 0x3)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0x69
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc420352880, 0x270d780, 0x2b84708, 0xc4216f58c0, 0x2702a00, 0x2b84708, 0x19d24a0, 0xc421e88a50, 0x7fffffff, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4216fc9a0, 0x19d24a0, 0xc421e88a50, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:386 +0x120
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc42026f7c0, 0x7ff51dc768f8, 0xc4210fe630, 0xc421384300)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:194 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x2715ee0, 0xc42026f7c0, 0xc4204a9520, 0x7ff51dc768f8, 0xc4210fe630, 0xc421384300, 0xc420b3d778, 0x66d25e)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:313 +0x35
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x7ff51dc768f8, 0xc4210fe630)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:321 +0xc0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42026fa80, 0xc421b79720, 0xc4211cdd80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66259 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4206b9520)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x666
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4206b9520)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x1d9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0x715

goroutine 66169 [select, 11 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc4200976c0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:742 +0x743
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:161 +0x896

goroutine 65970 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7ff51dc768f8, 0xc420abecc0)
    /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x181
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203061a0, 0xc4217a92c0, 0xc420306190)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65585 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x7ff51dc768f8, 0xc420282b70)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:113 +0x38b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42027f9d0, 0xc421b79720, 0xc4201e5400)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 67574 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42001a940, 0xc4201c3850, 0x5, 0x5, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x556
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc421003c20, 0xc4201c3850, 0x5, 0x5, 0x1fa95fe, 0x1abb87, 0x37)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x5c
io.ReadAtLeast(0x26fb040, 0xc421003c20, 0xc4201c3850, 0x5, 0x5, 0x5, 0x32, 0x32, 0x0)
    /usr/local/go/src/io/io.go:307 +0xa9
io.ReadFull(0x26fb040, 0xc421003c20, 0xc4201c3850, 0x5, 0x5, 0x0, 0x0, 0x0)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4201c3840, 0x7fffffff, 0x0, 0xc420ca0230, 0x37, 0xc4218bdaa0, 0x17, 0x6)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0x69
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4201c3840, 0x270d780, 0x2b84708, 0xc421003c20, 0x2702a00, 0x2b84708, 0x19e8b80, 0xc422627a40, 0x7fffffff, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc420aec8c0, 0x19e8b80, 0xc422627a40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:613 +0x11d
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc420320180, 0xc420aafbb8, 0xa155e2, 0xc422524540)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:230 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(Gossip_GossipServer).Recv-fm(0xc422524540, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:163 +0x2f
github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc422524500, 0x7ff51dc768f8, 0xc42037ce70, 0xc42000e540, 0xc42001aa00, 0xc420aaff60, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:341 +0x48b
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3.1(0x7ff51dc768f8, 0xc42037ce70)
    /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:163 +0x9f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203201c0, 0xc4217a92c0, 0xc42001aa80)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 93927 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func3(0x27152e0, 0xc4216fb680, 0xc421e92480, 0xc421b154a0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:236 +0x40f
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:256 +0xc94

goroutine 65827 [semacquire]:
sync.runtime_notifyListWait(0xc4223dd150, 0xc40000924d)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc4223dd140)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420397560, 0xc421b79720)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc420c45890)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420368280, 0xc421b79720, 0xc420246740)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66077 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc400008940)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc4202a9680)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb1b0, 0xc4217a92c0, 0xc4201fc660)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66094 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc400008935)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc4216289c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201bb2c0, 0xc4217a92c0, 0xc4201fc920)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65567 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7ff51dc768f8, 0xc422402180)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x364
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b8230, 0xc421b79720, 0xc4202dc200)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 66001 [semacquire]:
sync.runtime_notifyListWait(0xc420b4f510, 0xc400008948)
    /usr/local/go/src/runtime/sema.go:298 +0x10b
sync.(*Cond).Wait(0xc420b4f500)
    /usr/local/go/src/sync/cond.go:57 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421c6a1b0, 0xc4217a92c0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xac
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7ff51dc768f8, 0xc4202a9bc0)
    /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201baf80, 0xc4217a92c0, 0xc4201fc080)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad

goroutine 65726 [IO wait]:
net.runtime_pollWait(0x7ff51dd88200, 0x72, 0xe)
    /usr/local/go/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc42009c0d8, 0x72, 0x26feac0, 0x26f4690)
    /usr/local/go/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc42009c0d8, 0xc422364000, 0x8000)
    /usr/local/go/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).Read(0xc42009c070, 0xc422364000, 0x8000, 0x8000, 0x0, 0x26feac0, 0x26f4690)
    /usr/local/go/src/net/fd_unix.go:250 +0x1b7
net.(*conn).Read(0xc420c06010, 0xc422364000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:181 +0x70
bufio.(*Reader).Read(0xc420705320, 0xc42038e738, 0x9, 0x9, 0x0, 0x0, 0x0)
    /usr/local/go/src/bufio/bufio.go:213 +0x312
io.ReadAtLeast(0x26f6340, 0xc420705320, 0xc42038e738, 0x9, 0x9, 0x9, 0xc4209dba00, 0x1b7a1d8, 0x0)
    /usr/local/go/src/io/io.go:307 +0xa9
io.ReadFull(0x26f6340, 0xc420705320, 0xc42038e738, 0x9, 0x9, 0x26f6a40, 0xc420016100, 0x105000000000001)
    /usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42038e738, 0x9, 0x9, 0x26f6340, 0xc420705320, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42038e700, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc42216c4b0, 0xc4228c7260, 0xc4228c7260, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc420bac000)
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1043 +0xcd
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
    /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0xcc6

ERROR: exit status 2

make: *** [stress] Error 1
23 runs completed, 1 failures, over 30m1s
Makefile:197: recipe for target 'stress' failed
tamird commented 7 years ago

This contains https://github.com/cockroachdb/cockroach/pull/15555, unfortunately.

a-robinson commented 7 years ago

This is failing because the new TestCluster manual replication isn't waiting until node 3's replica has caught up with nodes 1 and 2 before returning success from AddReplicas. I'll fix this soon, but it's nothing more than a test problem and there are a couple things that need my attention more urgently so I'm going to skip it for now.