search

cockroachdb / cockroach

CockroachDB — the cloud native, distributed SQL database designed for high availability, effortless scale, and control over data placement.
https://www.cockroachlabs.com
Other
30.11k stars 3.81k forks source link

kv/kvserver: TestTenantRateLimiter failed #129963

Closed cockroach-teamcity closed 1 month ago

cockroach-teamcity commented 2 months ago

kv/kvserver.TestTenantRateLimiter failed with artifacts on release-23.1.26-rc @ 53c2718ea13bfe632da68d25c64182fdf9648d80:


goroutine 1391256 [sync.Cond.Wait]:
runtime.gopark(0xc0252685a0?, 0x0?, 0x0?, 0x0?, 0x0?)
    GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc03e949e00 sp=0xc03e949de0 pc=0x492a16
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:369
sync.runtime_notifyListWait(0xc01df2bd50, 0xaab)
    GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc03e949e48 sp=0xc03e949e00 pc=0x4c0a6c
sync.(*Cond).Wait(0x58a8980?)
    GOROOT/src/sync/cond.go:70 +0x8c fp=0xc03e949e80 sp=0xc03e949e48 pc=0x4dc22c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc021546a20, {0x7988aa0, 0xc018e9d920}, {0x796c2a0, 0xc01ada6a80}, 0xc01ada6000)
    github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:355 +0x278 fp=0xc03e949ee8 sp=0xc03e949e80 pc=0x231bb38
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7988aa0?, 0xc018e9d920?})
    github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:302 +0x45 fp=0xc03e949f30 sp=0xc03e949ee8 pc=0x231b485
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
    github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 fp=0xc03e949fe0 sp=0xc03e949f30 pc=0x1274a86
runtime.goexit()
    GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc03e949fe8 sp=0xc03e949fe0 pc=0x4c4bc1
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
    github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b

goroutine 1392549 [select, 1 minutes]:
runtime.gopark(0xc0037b56e8?, 0x3?, 0xf8?, 0x17?, 0xc0037b5642?)
    GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0037b5498 sp=0xc0037b5478 pc=0x492a16
runtime.selectgo(0xc0037b56e8, 0xc0037b563c, 0xc021419400?, 0x0, 0x6175ac0?, 0x1)
    GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0037b55d8 sp=0xc0037b5498 pc=0x4a2cfc
github.com/cockroachdb/cockroach/pkg/server.startSystemLogsGC.func1({0x7988aa0, 0xc00cf8fa70})
    github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:263 +0x245 fp=0xc0037b5730 sp=0xc0037b55d8 pc=0x42baca5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
    github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 fp=0xc0037b57e0 sp=0xc0037b5730 pc=0x1274a86
runtime.goexit()
    GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0037b57e8 sp=0xc0037b57e0 pc=0x4c4bc1
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
    github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b

goroutine 1392567 [select]:
runtime.gopark(0xc016a91f00?, 0x2?, 0xc0?, 0xb1?, 0xc016a91ed4?)
    GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc016a91d60 sp=0xc016a91d40 pc=0x492a16
runtime.selectgo(0xc016a91f00, 0xc016a91ed0, 0x0?, 0x0, 0x88b7c0?, 0x1)
    GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc016a91ea0 sp=0xc016a91d60 pc=0x4a2cfc
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc015009db0, 0x1)
    google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:416 +0x115 fp=0xc016a91f30 sp=0xc016a91ea0 pc=0xc57115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0156948a0)
    google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc016a91f88 sp=0xc016a91f30 pc=0xc57825
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
    google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:336 +0xce fp=0xc016a91fe0 sp=0xc016a91f88 pc=0xc6fb0e
runtime.goexit()
    GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc016a91fe8 sp=0xc016a91fe0 pc=0x4c4bc1
created by google.golang.org/grpc/internal/transport.NewServerTransport
    google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:333 +0x19b3

Parameters:

See also: How To Investigate a Go Test Failure (internal)

Same failure on other branches

- #129612 kv/kvserver: TestTenantRateLimiter failed [A-testing C-bug C-test-failure O-robot T-kv branch-release-24.1.4-rc] - #127697 kv/kvserver: TestTenantRateLimiter failed [A-testing C-bug C-test-failure O-robot P-3 T-kv branch-release-24.1]

/cc @cockroachdb/kv

This test on roachdash | Improve this report!

Jira issue: CRDB-41805

tbg commented 2 months ago

Looks like we entered near-infinite recursion in *rowsIterator. Sadly, the interesting frames are elided.

Details

``` === RUN TestTenantRateLimiter test_log_scope.go:161: test logs captured to: /artifacts/tmp/_tmp/33e1d369c27b9c01b2b6009c561815a3/logTestTenantRateLimiter3068297417 test_log_scope.go:79: use -show-logs to present logs inline runtime: goroutine stack exceeds 1000000000-byte limit runtime: sp=0xc04da00430 stack=[0xc04da00000, 0xc06da00000] fatal error: stack overflow runtime stack: runtime.throw({0x6262969?, 0xadf34e0?}) GOROOT/src/runtime/panic.go:1047 +0x5d fp=0x7ff511dfc278 sp=0x7ff511dfc248 pc=0x48fdbd runtime.newstack() GOROOT/src/runtime/stack.go:1103 +0x5cc fp=0x7ff511dfc430 sp=0x7ff511dfc278 pc=0x4aa6ac runtime.morestack() GOROOT/src/runtime/asm_amd64.s:570 +0x8b fp=0x7ff511dfc438 sp=0x7ff511dfc430 pc=0x4c2aeb goroutine 1397990 [running]: runtime.selectgo(0xc04da00480?, 0xc04da00474?, 0x0?, 0x0?, 0x3?, 0x1?) GOROOT/src/runtime/select.go:122 +0x1516 fp=0xc04da00440 sp=0xc04da00438 pc=0x4a3a56 github.com/cockroachdb/cockroach/pkg/sql.(*ieResultChannel).firstResult(0xc023a8b040, {0x7988aa0, 0xc001998090}) github.com/cockroachdb/cockroach/pkg/sql/internal_result_channel.go:123 +0xf9 fp=0xc04da00598 sp=0xc04da00440 pc=0x3ab9ef9 github.com/cockroachdb/cockroach/pkg/sql.(*ieResultChannel).nextResult(0x0?, {0x7988aa0, 0xc001998090}) github.com/cockroachdb/cockroach/pkg/sql/internal_result_channel.go:162 +0xfe fp=0xc04da006b8 sp=0xc04da00598 pc=0x3aba5be github.com/cockroachdb/cockroach/pkg/sql.(*rowsIterator).Next(0xc01027a000, {0x7988aa0?, 0xc001998090?}) github.com/cockroachdb/cockroach/pkg/sql/internal.go:482 +0x21a fp=0xc04da00840 sp=0xc04da006b8 pc=0x3ab21ba github.com/cockroachdb/cockroach/pkg/sql.(*rowsIterator).Next.func2({{0x0, 0x0, 0x0}, 0xc03a0bc058, {0x0, 0x0, 0x0}, {0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/sql/internal.go:450 +0x26c fp=0xc04da008e0 sp=0xc04da00840 pc=0x3ab268c github.com/cockroachdb/cockroach/pkg/sql.(*rowsIterator).Next(0xc01027a000, {0x7988aa0?, 0xc001998090?}) github.com/cockroachdb/cockroach/pkg/sql/internal.go:486 +0x39f fp=0xc04da00a68 sp=0xc04da008e0 pc=0x3ab233f github.com/cockroachdb/cockroach/pkg/sql.(*rowsIterator).Next.func2({{0x0, 0x0, 0x0}, 0xc03a0bc008, {0x0, 0x0, 0x0}, {0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/sql/internal.go:450 +0x26c fp=0xc04da00b08 sp=0xc04da00a68 pc=0x3ab268c github.com/cockroachdb/cockroach/pkg/sql.(*rowsIterator).Next(0xc01027a000, {0x7988aa0?, 0xc001998090?}) [...] github.com/cockroachdb/cockroach/pkg/sql.(*rowsIterator).Next(0xc01027a000, {0x7988aa0?, 0xc001998090?}) github.com/cockroachdb/cockroach/pkg/sql/internal.go:486 +0x39f fp=0xc04da06fc0 sp=0xc04da06e38 pc=0x3ab233f ...additional frames elided... created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b ```

But this stack seems to be waiting for a results channel and is related to a job:

Details

``` goroutine 1392680 [select, 1 minutes]: runtime.gopark(0xc01e6dc978?, 0x3?, 0xd8?, 0x21?, 0xc01e6dc972?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc02812c7f8 sp=0xc02812c7d8 pc=0x492a16 runtime.selectgo(0xc02812c978, 0xc01e6dc96c, 0x8?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc02812c938 sp=0xc02812c7f8 pc=0x4a2cfc github.com/cockroachdb/cockroach/pkg/sql.(*ieResultChannel).firstResult(0xc00bd40040, {0x7988aa0, 0xc00c7083f0}) github.com/cockroachdb/cockroach/pkg/sql/internal_result_channel.go:123 +0xf9 fp=0xc02812ca90 sp=0xc02812c938 pc=0x3ab9ef9 github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).execInternal(0xc0195ef180, {0x79889f8, 0xc0169417c0}, {0x62ad319, 0x19}, 0xc00bd40040, 0x1, 0xc0013d8630, {{{0x6227bf5, 0x4}}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/internal.go:1122 +0x1226 fp=0xc02812d068 sp=0xc02812ca90 pc=0x3ab56a6 github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).ExecEx(0x0?, {0x79889f8, 0xc0169417c0}, {0x62ad319, 0x19}, 0x0?, {{{0x6227bf5, 0x4}}, {0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/internal.go:692 +0x195 fp=0xc02812d168 sp=0xc02812d068 pc=0x3ab3bd5 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func6.1({0x79889f8, 0xc0169417c0}, {0x79e6520, 0xc0195ef180}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1075 +0x245 fp=0xc02812d280 sp=0xc02812d168 pc=0x27d4345 github.com/cockroachdb/cockroach/pkg/sql.(*internalDBWithOverrides).Txn.func1({0x79889f8, 0xc0169417c0}, {0x79e6520?, 0xc0195ef180?}) github.com/cockroachdb/cockroach/pkg/sql/internal.go:1791 +0xb3 fp=0xc02812d2e8 sp=0xc02812d280 pc=0x3ab9b93 github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).Txn.func1({0x79889f8?, 0xc0169417c0?}, 0x79e8848?) github.com/cockroachdb/cockroach/pkg/sql/internal.go:1639 +0x31 fp=0xc02812d318 sp=0xc02812d2e8 pc=0x3ab8a31 github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).txn.func4({0x79889f8, 0xc0169417c0}, 0xc0013d8630) github.com/cockroachdb/cockroach/pkg/sql/internal.go:1717 +0x3b4 fp=0xc02812d5c0 sp=0xc02812d318 pc=0x3ab9214 github.com/cockroachdb/cockroach/pkg/kv.runTxn.func1({0x79889f8, 0xc0169417c0}, 0xc00e9cb0e0?) github.com/cockroachdb/cockroach/pkg/kv/db.go:994 +0x2c fp=0xc02812d600 sp=0xc02812d5c0 pc=0x141a06c github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec(0xc0013d8630, {0x79889f8, 0xc0169417c0}, 0xc02812d960) github.com/cockroachdb/cockroach/pkg/kv/txn.go:928 +0xae fp=0xc02812d908 sp=0xc02812d600 pc=0x142266e github.com/cockroachdb/cockroach/pkg/kv.runTxn({0x79889f8, 0xc0169417c0}, 0xc0169417c0?, 0x0?) github.com/cockroachdb/cockroach/pkg/kv/db.go:993 +0x6b fp=0xc02812d9a0 sp=0xc02812d908 pc=0x1419eeb github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl(0x7ff53f9c21d8?, {0x79889f8, 0xc0169417c0}, 0xc8abfc0?, 0xc0?, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/kv/db.go:956 +0xa7 fp=0xc02812d9f0 sp=0xc02812d9a0 pc=0x1419c47 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Txn(0x463547?, {0x79889f8?, 0xc0169417c0?}, 0xc00fc67a01?) github.com/cockroachdb/cockroach/pkg/kv/db.go:931 +0x2d fp=0xc02812da28 sp=0xc02812d9f0 pc=0x1419b4d github.com/cockroachdb/cockroach/pkg/kv.(*DB).Txn-fm({0x79889f8?, 0xc0169417c0?}, 0x79889f8?) :1 +0x3c fp=0xc02812da58 sp=0xc02812da28 pc=0x3c497fc github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).txn(0xc00d0b58c0, {0x79889f8, 0xc0169417c0}, 0xc011bb87f0, {0x0, 0x0, 0x5c10060?}) github.com/cockroachdb/cockroach/pkg/sql/internal.go:1705 +0x2a7 fp=0xc02812db38 sp=0xc02812da58 pc=0x3ab8d27 github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).Txn(0x0?, {0x79889f8, 0xc0169417c0}, 0xc0095ccee8, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/internal.go:1640 +0x9f fp=0xc02812db88 sp=0xc02812db38 pc=0x3ab899f github.com/cockroachdb/cockroach/pkg/sql.(*internalDBWithOverrides).Txn(0xc00ea073b0, {0x79889f8, 0xc0169417c0}, 0xc0095cced0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/internal.go:1787 +0xbd fp=0xc02812dbd0 sp=0xc02812db88 pc=0x3ab9a7d github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func6({0x79889f8, 0xc0169417c0}, {0x7979c50?, 0xc0090d7880}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1068 +0xc5 fp=0xc02812dc50 sp=0xc02812dbd0 pc=0x27d4045 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func7({0x79889f8, 0xc0169417c0}, {0x7979c50, 0xc0090d7880}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1097 +0xe7 fp=0xc02812dcd8 sp=0xc02812dc50 pc=0x27d3e87 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).withSession(0xc007270f00, {0x79889f8, 0xc0169417c0}, 0xc009ca9120) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:996 +0x26a fp=0xc02812dda0 sp=0xc02812dcd8 pc=0x27d26ca github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func13({0x79889f8?, 0xc0169417c0?}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1012 +0x34 fp=0xc02812ddd0 sp=0xc02812dda0 pc=0x27d3d74 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10({0x7988aa0?, 0xc00e77e000?}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1204 +0x2a2 fp=0xc02812df30 sp=0xc02812ddd0 pc=0x27d3382 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 fp=0xc02812dfe0 sp=0xc02812df30 pc=0x1274a86 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc02812dfe8 sp=0xc02812dfe0 pc=0x4c4bc1 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b ```

and there are a few more stacks in SQL, test output attached.

test.log.txt