search

cockroachdb / cockroach

CockroachDB — the cloud native, distributed SQL database designed for high availability, effortless scale, and control over data placement.
https://www.cockroachlabs.com
Other
30.02k stars 3.79k forks source link

kv/kvserver: TestStoreRangeMergeTimestampCache failed #66932

Closed cockroach-teamcity closed 2 years ago

cockroach-teamcity commented 3 years ago

kv/kvserver.TestStoreRangeMergeTimestampCache failed with artifacts on master @ a142e392ddebb2d8a7c575170ac168cdefa90088:

Fatal error:

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
    panic: runtime error: invalid memory address or nil pointer dereference [recovered]
    panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x2d48806]

Stack: 

goroutine 117256 [running]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send.func1(0xc0070296b8, 0xc007029770, 0xc003dd8800, 0xc007029768)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:85 +0x725
panic(0x5ea2520, 0xb16e950)
    /usr/local/go/src/runtime/panic.go:971 +0x499
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Recover(0xc0062310e0, 0x920f238, 0xc006439200)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:233 +0x15d
panic(0x5ea2520, 0xb16e950)
    /usr/local/go/src/runtime/panic.go:971 +0x499
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send.func1(0xc00701f2a8, 0xc00701f360, 0xc003dd8800, 0xc00701f358)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:98 +0x366
runtime.Goexit()
    /usr/local/go/src/runtime/panic.go:632 +0x146
testing.(*common).FailNow(0xc000f7d080)
    /usr/local/go/src/testing/testing.go:741 +0x65
testing.(*common).Fatalf(0xc000f7d080, 0x6546bab, 0x2d, 0xc006fd24e0, 0x3, 0x3)
    /usr/local/go/src/testing/testing.go:815 +0x9e
github.com/cockroachdb/cockroach/pkg/testutils.SucceedsSoon(0x9236d78, 0xc000f7d080, 0xc00701e350)
    /go/src/github.com/cockroachdb/cockroach/pkg/testutils/soon.go:34 +0x1ea
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).WaitForValues(0xc001340480, 0x9349d50, 0xc000f7d080, 0xc00b36c3f0, 0x1, 0x1, 0xc005da7fb0, 0x3, 0x3)
    /go/src/github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:1319 +0x110
github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.mergeCheckingTimestampCaches.func9()
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:691 +0x7ec
github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.mergeCheckingTimestampCaches.func1(0x920f238, 0xc006439350, 0x168c5b9a8c8fb913, 0x2, 0x100000001, 0x1, 0x0, 0x5, 0x0, 0xc00100c6c0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:436 +0x192
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc007f3a000, 0x920f238, 0xc006439230, 0x5, 0xc007ac7ad0, 0x2, 0x1)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:93 +0x8b9
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:43
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc003dd8800, 0x920f238, 0xc006439200, 0x168c5b9a8c8fb913, 0x2, 0x100000001, 0x1, 0x0, 0x5, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:180 +0x5b3
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc00a93a580, 0x920f238, 0xc006439200, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x198
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x920f238, 0xc006439200, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:890 +0x27e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc0062310e0, 0x920f238, 0xc006439200, 0x64c5095, 0x10, 0xc00701f978, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:326 +0x12a
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc002314c00, 0x920f238, 0xc006439200, 0xc007ac7a20, 0x0, 0x0, 0x1)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:878 +0x267
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc002314c00, 0x920f238, 0xc0064391d0, 0xc007ac7a20, 0x5b159c, 0xc006439080, 0x4a2c8c)
    /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:945 +0x106
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
    /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:462
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc006891dc0, 0x920f238, 0xc0064391d0, 0x1, 0x9255ad0, 0xc001b191b0, 0x0, 0x0, 0x100000001, 0x1, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x25d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc006891dc0, 0x920f238, 0xc006439080, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x25f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.raceTransport.SendNext(0x9259ee8, 0xc006891dc0, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport_race.go:78 +0x443
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc0054a7400, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1875 +0xe75
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc0054a7400, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1493 +0x410
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc0054a7400, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1116 +0x1f5f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc0054a7400, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:755 +0x77b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc002cc9860, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x1b5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc002cc9828, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0xfb
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc002cc97f8, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:200 +0x275
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc002cc9758, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:240 +0x115
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc002cc9758, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:175 +0x36a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc002cc9688, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x295
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc002cc9668, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x355
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc002cc95c8, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:241 +0x275
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc002cc9400, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x575
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00a93a400, 0x920f238, 0xc009069f80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:817 +0x175
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc002c4e790, 0x920f238, 0xc009069f80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:982 +0x228
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x920f238, 0xc009069f80, 0xc007027be0, 0xc002cdc000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:728 +0x1d9
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc002c4e790, 0x920f238, 0xc009069f80, 0xc002cdc000, 0xc001b55800, 0x3)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:635 +0xd7
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).AdminMerge.func1(0xc002c4e790, 0xc009069f80, 0xc00a93a400)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_command.go:764 +0x1c90
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).AdminMerge(0xc007f3a000, 0x920f238, 0xc009069f80, 0xc000d271d8, 0x4, 0x8, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_command.go:785 +0x1ef
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeAdminBatch(0xc007f3a000, 0x920f238, 0xc009069f80, 0xc002c4e630, 0x100000001, 0x1)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:706 +0x1486
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc007f3a000, 0x920f238, 0xc003234ab0, 0x5, 0xc002c4e630, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:110 +0x6f1
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:43
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc003dd8800, 0x920f1c8, 0xc000074108, 0x168c5b9a8c8fb913, 0x0, 0x100000001, 0x1, 0x0, 0x5, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:180 +0x5b3
github.com/cockroachdb/cockroach/pkg/kv.Wrap.func1(0x920f1c8, 0xc000074108, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:456 +0x13b
github.com/cockroachdb/cockroach/pkg/kv.SenderFunc.Send(0xc00671afc0, 0x920f1c8, 0xc000074108, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:392 +0xb9
github.com/cockroachdb/cockroach/pkg/kv.SendWrappedWith(0x920f1c8, 0xc000074108, 0x918ff20, 0xc00671afc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:432 +0x17b
github.com/cockroachdb/cockroach/pkg/kv.SendWrapped(...)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:449
github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.mergeCheckingTimestampCaches.func11(0xc002ebda20, 0x920f1c8, 0xc000074108, 0xc003dd8800, 0xc004033aa0)
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:751 +0x19a
created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.mergeCheckingTimestampCaches
    /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:749 +0x22cc
Log preceding fatal error

``` github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc002cc9860, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x1b5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc002cc9828, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0xfb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc002cc97f8, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:200 +0x275 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc002cc9758, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:240 +0x115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc002cc9758, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:175 +0x36a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc002cc9688, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x295 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc002cc9668, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x355 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc002cc95c8, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:241 +0x275 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc002cc9400, 0x920f238, 0xc006439080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x575 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00a93a400, 0x920f238, 0xc009069f80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:817 +0x175 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc002c4e790, 0x920f238, 0xc009069f80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:982 +0x228 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x920f238, 0xc009069f80, 0xc007027be0, 0xc002cdc000, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:728 +0x1d9 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc002c4e790, 0x920f238, 0xc009069f80, 0xc002cdc000, 0xc001b55800, 0x3) /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:635 +0xd7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).AdminMerge.func1(0xc002c4e790, 0xc009069f80, 0xc00a93a400) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_command.go:764 +0x1c90 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).AdminMerge(0xc007f3a000, 0x920f238, 0xc009069f80, 0xc000d271d8, 0x4, 0x8, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_command.go:785 +0x1ef github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeAdminBatch(0xc007f3a000, 0x920f238, 0xc009069f80, 0xc002c4e630, 0x100000001, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:706 +0x1486 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc007f3a000, 0x920f238, 0xc003234ab0, 0x5, 0xc002c4e630, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:110 +0x6f1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:43 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc003dd8800, 0x920f1c8, 0xc000074108, 0x168c5b9a8c8fb913, 0x0, 0x100000001, 0x1, 0x0, 0x5, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:180 +0x5b3 github.com/cockroachdb/cockroach/pkg/kv.Wrap.func1(0x920f1c8, 0xc000074108, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:456 +0x13b github.com/cockroachdb/cockroach/pkg/kv.SenderFunc.Send(0xc00671afc0, 0x920f1c8, 0xc000074108, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:392 +0xb9 github.com/cockroachdb/cockroach/pkg/kv.SendWrappedWith(0x920f1c8, 0xc000074108, 0x918ff20, 0xc00671afc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:432 +0x17b github.com/cockroachdb/cockroach/pkg/kv.SendWrapped(...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:449 github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.mergeCheckingTimestampCaches.func11(0xc002ebda20, 0x920f1c8, 0xc000074108, 0xc003dd8800, 0xc004033aa0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:751 +0x19a created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.mergeCheckingTimestampCaches /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:749 +0x22cc ```

Reproduce

To reproduce, try: ```bash make stressrace TESTS=TestStoreRangeMergeTimestampCache PKG=./pkg/kv/kvserver TESTTIMEOUT=5m STRESSFLAGS='-timeout 5m' 2>&1 ```

Parameters in this failure: - GOFLAGS=-race -parallel=4

Internal log

``` benesch marked as alumn{us/a}; resolving to nvanbenschoten instead ```

/cc @cockroachdb/kv nvanbenschoten

This test on roachdash | Improve this report!

tbg commented 3 years ago

We NPEed on this line

https://github.com/cockroachdb/cockroach/blob/cbc0810ae10747e169dbb4f54fd0163594dedf42/pkg/kv/kvserver/store_send.go#L98

meaning that br == nil despite pErr == nil.

The problem here is that we've injected a requester filter that can end up calling t.Fatal, in which case we're unwinding without either setting an error or a result. This is a problem in the test but arguably also in the code (if we hit a "natural" NPE in the same place, the result would be roughly the same, i.e. we'd panic again - not an issue since we currently don't handle panics in that code path, but could become one as we get better about it).

tbg commented 3 years ago

Hmm, that test does lots of channel acrobatics. Hard to believe that this test would fail gracefully on a failure of any of its more interesting assertions. I'll see if there's anything that can be done here to make such tests less fragile.

cockroach-teamcity commented 3 years ago

kv/kvserver.TestStoreRangeMergeTimestampCache failed with artifacts on master @ 86763c0b2df8f1c1d91848e00bb29f0675bc489e:

=== RUN   TestStoreRangeMergeTimestampCache
    test_log_scope.go:73: test logs captured to: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestStoreRangeMergeTimestampCache521970756
    test_log_scope.go:74: use -show-logs to present logs inline
=== CONT  TestStoreRangeMergeTimestampCache
    client_merge_test.go:411: -- test log scope end --
test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestStoreRangeMergeTimestampCache521970756
--- FAIL: TestStoreRangeMergeTimestampCache (102.30s)
=== RUN   TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true/through-snapshot=true/future-read=true
    client_merge_test.go:758: condition failed to evaluate within 45s: merge not applied
        goroutine 136401 [running]:
        runtime/debug.Stack(0xc00b016c70, 0x91d4500, 0xc00945f8d8)
            /usr/local/go/src/runtime/debug/stack.go:24 +0xab
        github.com/cockroachdb/cockroach/pkg/testutils.SucceedsSoon(0x927ddd8, 0xc006144c00, 0xc00b016c70)
            /go/src/github.com/cockroachdb/cockroach/pkg/testutils/soon.go:35 +0x87
        github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.mergeCheckingTimestampCaches(0xc006144c00, 0x10101)
            /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:758 +0x2365
        github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeTimestampCache.func1.1.1(0xc006144c00, 0x1)
            /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:407 +0x5c
        github.com/cockroachdb/cockroach/pkg/testutils.RunTrueAndFalse.func1(0xc006144c00)
            /go/src/github.com/cockroachdb/cockroach/pkg/testutils/subtest.go:23 +0x6f
        testing.tRunner(0xc006144c00, 0xc005afe0d8)
            /usr/local/go/src/testing/testing.go:1193 +0x203
        created by testing.(*T).Run
            /usr/local/go/src/testing/testing.go:1238 +0x5d8
E210713 02:36:50.945761 139464 kv/kvserver/queue.go:1093  [n2,replicaGC,s2,r5/2:{/Systemtse-b}] 4  failed to send RPC: sending to all replicas failed; last error: failed to connect to n3 at 127.0.0.1:37433: stopped
E210713 02:36:50.953677 139273 kv/kvserver/consistency_queue.go:188  [n1,consistencyChecker,s1,r5/1:{/Systemtse-b}] 5  result is ambiguous (server shutdown)
E210713 02:36:50.955247 139273 kv/kvserver/queue.go:1093  [n1,consistencyChecker,s1,r5/1:{/Systemtse-b}] 6  result is ambiguous (server shutdown)
E210713 02:36:50.955770 139520 kv/kvserver/queue.go:1093  [n3,replicaGC,s3,r40/3:{b-/Table/Syste…}] 7  failed to send RPC: sending to all replicas failed; last error: rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:34544->127.0.0.1:43093: use of closed network connection
            --- FAIL: TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true/through-snapshot=true/future-read=true (49.88s)
=== RUN   TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true/through-snapshot=true
        --- FAIL: TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true/through-snapshot=true (60.94s)
=== RUN   TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true
    --- FAIL: TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true (69.79s)
Reproduce

To reproduce, try: ```bash make stressrace TESTS=TestStoreRangeMergeTimestampCache PKG=./pkg/kv/kvserver TESTTIMEOUT=5m STRESSFLAGS='-timeout 5m' 2>&1 ```

Parameters in this failure: - GOFLAGS=-json

Internal log

``` benesch marked as alumn{us/a}; resolving to nvanbenschoten instead ```

/cc @cockroachdb/kv nvanbenschoten

This test on roachdash | Improve this report!

cockroach-teamcity commented 3 years ago

kv/kvserver.TestStoreRangeMergeTimestampCache failed with artifacts on master @ cdace509b79339ba6c60f56e1bbe5d0b261de943:

=== RUN   TestStoreRangeMergeTimestampCache
    test_log_scope.go:73: test logs captured to: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestStoreRangeMergeTimestampCache396188320
    test_log_scope.go:74: use -show-logs to present logs inline
=== CONT  TestStoreRangeMergeTimestampCache
    client_merge_test.go:427: -- test log scope end --
test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestStoreRangeMergeTimestampCache396188320
--- FAIL: TestStoreRangeMergeTimestampCache (45.37s)
=== RUN   TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true/through-snapshot=false
        --- FAIL: TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true/through-snapshot=false (9.05s)
=== RUN   TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true
    --- FAIL: TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true (24.75s)
=== RUN   TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true/through-snapshot=false/future-read=true
    client_merge_test.go:600: 
            Error Trace:    client_merge_test.go:600
                                        client_merge_test.go:423
                                        subtest.go:23
            Error:          Expected nil, but got: &roachpb.Error{deprecatedMessage:"kv/kvserver/replica_command.go:791: merge failed: waiting for all right-hand replicas to catch up: operation \"wait for application\" timed out after 5s: rpc error: code = DeadlineExceeded desc = context deadline exceeded", deprecatedTransactionRestart:0, UnexposedTxn:(*roachpb.Transaction)(nil), OriginNode:0, deprecatedDetail:roachpb.ErrorDetail{Value:roachpb.isErrorDetail_Value(nil)}, EncodedError:errorspb.EncodedError{Error:(*errorspb.EncodedError_Wrapper)(0xc002098958)}, Index:(*roachpb.ErrPosition)(nil), Now:hlc.ClockTimestamp{WallTime:1627458811546891387, Logical:514, Synthetic:false}}
            Test:           TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true/through-snapshot=false/future-read=true
E210728 07:53:36.566028 149928 kv/kvserver/queue.go:1098  [n3,replicaGC,s3,r42/3:{\xfab-/Max}] 1  failed to send RPC: sending to all replicas failed; last error: rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:53812->127.0.0.1:41127: use of closed network connection
E210728 07:53:36.567285 149805 kv/kvserver/queue.go:1098  [n2,replicaGC,s2,r42/2:{\xfab-/Max}] 2  failed to send RPC: sending to all replicas failed; last error: rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:53790->127.0.0.1:41127: use of closed network connection
E210728 07:53:36.569368 149830 kv/kvserver/consistency_queue.go:188  [n2,consistencyChecker,s2,r42/2:{\xfab-/Max}] 3  failed to send RPC: sending to all replicas failed; last error: failed to connect to n3 at 127.0.0.1:40575: context canceled
E210728 07:53:36.569454 149830 kv/kvserver/queue.go:1098  [n2,consistencyChecker,s2,r42/2:{\xfab-/Max}] 4  failed to send RPC: sending to all replicas failed; last error: failed to connect to n3 at 127.0.0.1:40575: context canceled
            --- FAIL: TestStoreRangeMergeTimestampCache/disjoint-leaseholders=true/through-snapshot=false/future-read=true (6.61s)
Reproduce

To reproduce, try: ```bash make stressrace TESTS=TestStoreRangeMergeTimestampCache PKG=./pkg/kv/kvserver TESTTIMEOUT=5m STRESSFLAGS='-timeout 5m' 2>&1 ```

Parameters in this failure: - GOFLAGS=-json

Internal log

``` benesch marked as alumn{us/a}; resolving to nvanbenschoten instead ```

/cc @cockroachdb/kv nvanbenschoten

This test on roachdash | Improve this report!

tbg commented 2 years ago

Not skipped and not flaky, closing as stale