cockroach-teamcity
commented
1 year ago ccl/streamingccl/streamingest.TestAlterTenantHandleFutureProtectedTimestamp failed with artifacts on release-23.1 @ cd51cf8bef3638deb82d32f0430447bdbd1a97ae:
Fatal error:
panic: test timed out after 14m55s
Stack:
goroutine 342183 [running]:
testing.(*M).startAlarm.func1()
GOROOT/src/testing/testing.go:2036 +0x88
created by time.goFunc
GOROOT/src/time/sleep.go:176 +0x38Log preceding fatal error
``` * bufio.(*Reader).fill(0x4009273cb0) * GOROOT/src/bufio/bufio.go:106 +0xfc * bufio.(*Reader).ReadByte(0x4009273cb0) * GOROOT/src/bufio/bufio.go:265 +0x2c * github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0x40092742c0?, {0x6b1a710, 0x4009273cb0}) * github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:214 +0x30 * github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0x4009273c00, 0x400ce2b8b0, 0x400d3ecd55, 0x400d3ecd57, {0x6b2deb0, 0x400e799a40}, 0x400de40e10, 0x400d3ecd53) * github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:438 +0x68 * github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0x4009273c00, {0x6b3eb40, 0x400dd40c00}, 0x400ce2b8a0, 0x4003b82000, 0x400dd40ba0, {0x0, 0x4, {0x1, {0x553ffb4, ...}, ...}, ...}) * github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:576 +0x920 * github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0x4003433380, {0x6b3eb40, 0x400dd40c00}, {0x6b83308, 0x400ddb5180}, {{{0x4011591031, 0x4}}, 0x1, 0x1, {{0x0, ...}}, ...}, ...) * github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:178 +0x210 * github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0x4003433380, {0x6b3eb40, 0x400dd40c00}, {0x6b83308, 0x400ddb5180}, {0x0, 0x4, 0x0, {0x5400, 0x15000, ...}, ...}) * github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:815 +0x924 * github.com/cockroachdb/cockroach/pkg/server.(*systemServerWrapper).serveConn(0x4000974a78, {0x6b3eb40, 0x400f182360}, {0x6b83308, 0x400ddb5180}, {0x0, 0x4, 0x0, {0x5400, 0x15000, ...}, ...}) * github.com/cockroachdb/cockroach/pkg/server/server_controller_sql.go:93 +0x100 * github.com/cockroachdb/cockroach/pkg/server.(*serverController).sqlMux(0x40090cc630, {0x6b3eb40, 0x400f182360}, {0x6b83308, 0x400ddb5180}, {0x0, 0x4, 0x0, {0x5400, 0x15000, ...}, ...}) * github.com/cockroachdb/cockroach/pkg/server/server_controller_sql.go:73 +0x218 * github.com/cockroachdb/cockroach/pkg/server.startServeSQL.func1.1({0x6b3eb40?, 0x400f182330?}, {0x6b89ef8, 0x4002d6a578}) * github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1770 +0x1a0 * github.com/cockroachdb/cockroach/pkg/util/netutil.(*TCPServer).ServeWith.func1({0x6b3eb40, 0x400f182330}) * github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:188 +0xa4 * github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() * github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x128 * created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx * github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x358 * * goroutine 112034 [select]: * github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x6b3eb40, 0x400b075890}) * github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0xc0 * github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() * github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x128 * created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx * github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x358 * * goroutine 111924 [sync.Cond.Wait]: * sync.runtime_notifyListWait(0x4004740450, 0x1a7) * GOROOT/src/runtime/sema.go:517 +0x160 * sync.(*Cond).Wait(0x4004740440) * GOROOT/src/sync/cond.go:70 +0xd0 * github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0x4001c0b8c0, {0x6b3eb40, 0x40009aa4b0}, {0x6b22900, 0x400392d500}, 0x4001216400) * github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:355 +0x2c8 * github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6b3eb40?, 0x40009aa4b0?}) * github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:302 +0x48 * github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() * github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x128 * created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx * github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x358 * * ```
Help
See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
ccl/streamingccl/streamingest.TestAlterTenantHandleFutureProtectedTimestamp failed with artifacts on release-23.1 @ afe0240f6266af0b21fb521bd6120858ae99f442:
Fatal error:
Stack:
Log preceding fatal error
``` * github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc001d0ca80, 0xc000263e00, {0x73daa80, 0xc00992c3b0}) * github.com/cockroachdb/cockroach/pkg/server/node.go:1472 +0x1ab * github.com/cockroachdb/cockroach/pkg/kv/kvpb._Internal_RangeFeed_Handler({0x5d16b00?, 0xc001d0ca80}, {0x73d54b8, 0xc000c73840}) * github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:10062 +0xd0 * github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5d16b00, 0xc001d0ca80}, {0x73d1768?, 0xc002c5f800?}, 0xc00c304d38, 0x617c010) * github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 * google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5d16b00?, 0xc001d0ca80?}, {0x73d1768?, 0xc002c5f800?}) * google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b * github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5d16b00, 0xc001d0ca80}, {0x73d1768, 0xc002c5f800}, 0x5a779e0?, 0xc006804980) * github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74 * google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5d16b00?, 0xc001d0ca80?}, {0x73d1768?, 0xc002c5f800?}) * google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 * github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0009e8000?, {{0xc0046220e8?}, {0x73d1690?, 0xc008d3bf20?}}}, {0x5d16b00, 0xc001d0ca80}, {0x73d5d28, 0xc00be76700}, 0xc00c304d38, 0xc006804980) * github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7 * google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5d16b00?, 0xc001d0ca80?}, {0x73d5d28?, 0xc00be76700?}) * google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 * github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) * github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d * github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000bc1b00, {0x739e620, 0xc002c5f740}, {0xc0049b5a98?, 0x4631ff?}, 0xc0049b5a50) * github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1 * github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5d16b00, 0xc001d0ca80}, {0x73d5d28?, 0xc00be76700?}, 0xc00c304d38, 0xc006804980) * github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef * google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5d16b00?, 0xc001d0ca80?}, {0x73d5d28?, 0xc00be76700?}) * google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 * google.golang.org/grpc.chainStreamInterceptors.func1({0x5d16b00, 0xc001d0ca80}, {0x73d5d28, 0xc00be76700}, 0xc00c304d38, 0x617c010) * google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b * google.golang.org/grpc.(*Server).processStreamingRPC(0xc007780d20, {0x73ee460, 0xc00425c9c0}, 0xc00279e120, 0xc00b094990, 0xa489800, 0x0) * google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6 * google.golang.org/grpc.(*Server).handleStream(0xc007780d20, {0x73ee460, 0xc00425c9c0}, 0xc00279e120, 0x0) * google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea * google.golang.org/grpc.(*Server).serveStreams.func1.2() * google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98 * created by google.golang.org/grpc.(*Server).serveStreams.func1 * google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a * * goroutine 109033 [select]: * github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() * github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6 * created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs * github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56 * * goroutine 112930 [select, 2 minutes]: * github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc002d2e800, 0x0}) * github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 * github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() * github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 * created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx * github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b * * ```
Help
See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
/cc @cockroachdb/disaster-recovery
This test on roachdash | Improve this report!
Jira issue: CRDB-28289