Open okJiang opened 3 weeks ago
panic: test timed out after 5m0s running tests: TestWatcher (5m0s) goroutine 368 [running]: testing.(*M).startAlarm.func1() /opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:2259 +0x259 created by time.goFunc /opt/hostedtoolcache/go/1.21.13/x64/src/time/sleep.go:176 +0x45 goroutine 1 [chan receive, 5 minutes]: testing.(*T).Run(0xc000281040, {0x20179fb, 0xb}, 0x20a8470) /opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1649 +0x871 testing.runTests.func1(0x0?) /opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:2054 +0x85 testing.tRunner(0xc000281040, 0xc0005dbae8) /opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1595 +0x262 testing.runTests(0xc0002ddae0?, {0x2e9f840, 0x2a, 0x2a}, {0xd0?, 0x1e?, 0x2eb0040?}) /opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:2052 +0x8ae testing.(*M).Run(0xc0002ddae0) /opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1925 +0xcd8 main.main() _testmain.go:163 +0x2e5 goroutine 179 [select, 5 minutes]: go.etcd.io/etcd/server/v3/mvcc.(*watchableStore).syncVictimsLoop(0xc0002dc640) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/mvcc/watchable_store.go:261 +0x233 created by go.etcd.io/etcd/server/v3/mvcc.newWatchableStore in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/mvcc/watchable_store.go:98 +0xc25 goroutine 175 [select, 5 minutes]: go.etcd.io/etcd/pkg/v3/schedule.(*fifo).run(0xc00065b2c0) /home/runner/go/pkg/mod/go.etcd.io/etcd/pkg/v3@v3.5.15/schedule/schedule.go:143 +0x42d created by go.etcd.io/etcd/pkg/v3/schedule.NewFIFOScheduler in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/pkg/v3@v3.5.15/schedule/schedule.go:70 +0x28a goroutine 172 [select]: go.etcd.io/etcd/raft/v3.(*node).run(0xc00065afc0) /home/runner/go/pkg/mod/go.etcd.io/etcd/raft/v3@v3.5.15/node.go:341 +0xc5d created by go.etcd.io/etcd/raft/v3.StartNode in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/raft/v3@v3.5.15/node.go:230 +0x306 goroutine 170 [select]: go.etcd.io/etcd/server/v3/mvcc/backend.(*backend).run(0xc0002c4b40) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/mvcc/backend/backend.go:433 +0x269 created by go.etcd.io/etcd/server/v3/mvcc/backend.newBackend in goroutine 169 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/mvcc/backend/backend.go:241 +0xd56 goroutine 198 [select]: go.etcd.io/etcd/server/v3/etcdserver.(*raftNode).start.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/raft.go:174 +0x25d created by go.etcd.io/etcd/server/v3/etcdserver.(*raftNode).start in goroutine [182](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:183) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/raft.go:169 +0xea goroutine 174 [select]: go.etcd.io/etcd/server/v3/lease.(*lessor).runLoop(0xc0002c4e40) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/lease/lessor.go:616 +0x1c9 created by go.etcd.io/etcd/server/v3/lease.newLessor in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/lease/lessor.go:237 +0x5a5 goroutine 162 [chan receive, 5 minutes]: github.com/tikv/pd/pkg/encryption.TestWatcher(0xc000281380) /home/runner/work/pd/pd/pkg/encryption/key_manager_test.go:362 +0x914 testing.tRunner(0xc000281380, 0x20a8470) /opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1595 +0x262 created by testing.(*T).Run in goroutine 1 /opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1648 +0x846 goroutine 171 [select, 5 minutes]: go.etcd.io/etcd/server/v3/wal.(*filePipeline).run(0xc000471300) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/wal/file_pipeline.go:97 +0x205 created by go.etcd.io/etcd/server/v3/wal.newFilePipeline in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/wal/file_pipeline.go:55 +0x4a9 goroutine 178 [select]: go.etcd.io/etcd/server/v3/mvcc.(*watchableStore).syncWatchersLoop(0xc0002dc640) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/mvcc/watchable_store.go:235 +0x277 created by go.etcd.io/etcd/server/v3/mvcc.newWatchableStore in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/mvcc/watchable_store.go:97 +0xbbb goroutine 200 [select]: go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/v3@v3.5.15/fileutil/purge.go:104 +0x4c5 created by go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile in goroutine [185](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:186) /home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/v3@v3.5.15/fileutil/purge.go:56 +0x894 goroutine 182 [select]: go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).run(0xc00033ea00) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:1141 +0xbe5 created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).start in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:877 +0x159a goroutine 197 [select]: go.etcd.io/etcd/pkg/v3/schedule.(*fifo).run(0xc000541140) /home/runner/go/pkg/mod/go.etcd.io/etcd/pkg/v3@v3.5.15/schedule/schedule.go:143 +0x42d created by go.etcd.io/etcd/pkg/v3/schedule.NewFIFOScheduler in goroutine 182 /home/runner/go/pkg/mod/go.etcd.io/etcd/pkg/v3@v3.5.15/schedule/schedule.go:70 +0x28a goroutine 231 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000108370, 0x1) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:418 +0x1af google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003558f0) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:551 +0x14f google.golang.org/grpc/internal/transport.NewServerTransport.func2() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:325 +0x269 created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 230 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:322 +0x260b goroutine 185 [select, 5 minutes]: go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).purgeFile(0xc00033ea00) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:892 +0x485 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2812 +0x96 created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2810 +0x1cb goroutine [186](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:187) [select, 5 minutes]: go.etcd.io/etcd/server/v3/etcdserver.monitorFileDescriptor(0xc00033ea00?, 0xc0002b8720) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/metrics.go:237 +0x3fa go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).Start.func3() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:817 +0x65 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2812 +0x96 created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2810 +0x1cb goroutine [187](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:188) [select]: go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).monitorVersions(0xc00033ea00) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2542 +0x145 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2812 +0x96 created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2810 +0x1cb goroutine [188](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:189) [select]: go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).linearizableReadLoop(0xc00033ea00) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/v3_server.go:818 +0x171 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2812 +0x96 created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2810 +0x1cb goroutine 199 [select]: go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/v3@v3.5.15/fileutil/purge.go:104 +0x4c5 created by go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile in goroutine 185 /home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/v3@v3.5.15/fileutil/purge.go:56 +0x894 goroutine 204 [IO wait, 5 minutes]: internal/poll.runtime_pollWait(0x7f525c358c38, 0x72) /opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc00023cca0, 0x4af001?, 0x0) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1 internal/poll.(*pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc00023cc80) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:611 +0x425 net.(*netFD).accept(0xc00023cc80) /opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_unix.go:172 +0x3e net.(*TCPListener).accept(0xc0003083e0) /opt/hostedtoolcache/go/1.21.13/x64/src/net/tcpsock_posix.go:152 +0x3e net.(*TCPListener).Accept(0xc0003083e0) /opt/hostedtoolcache/go/1.21.13/x64/src/net/tcpsock.go:315 +0x65 go.etcd.io/etcd/client/pkg/v3/transport.(*keepaliveListener).Accept(0xc00044e1e0) /home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/v3@v3.5.15/transport/keepalive_listener.go:52 +0x53 go.etcd.io/etcd/client/pkg/v3/transport.(*limitListener).Accept(0xc00051e630) /home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/v3@v3.5.15/transport/limit_listen.go:46 +0x6d github.com/soheilhy/cmux.(*cMux).Serve(0xc000307c70) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/cmux.go:170 +0x13c go.etcd.io/etcd/server/v3/embed.(*serveCtx).serve(0xc00023cc00, 0xc00033ea00, 0xc00033c880, {0x231b880, 0xc000564e40}, 0xc000598870, 0xc0005edf18?, 0x0, {0xc0001544e0, 0x2, ...}) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/serve.go:257 +0x2b6b go.etcd.io/etcd/server/v3/embed.(*Etcd).serveClients.func1(0x0?) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/etcd.go:782 +0x1f0 created by go.etcd.io/etcd/server/v3/embed.(*Etcd).serveClients in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/etcd.go:781 +0x1412 goroutine 191 [select]: go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).monitorDowngrade(0xc00033ea00) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2708 +0x155 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2812 +0x96 created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2810 +0x1cb goroutine 192 [select]: go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.monitorLeader.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/interceptor.go:316 +0x173 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2812 +0x96 created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2810 +0x1cb goroutine 193 [select, 5 minutes]: github.com/soheilhy/cmux.muxListener.Accept(...) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/cmux.go:262 google.golang.org/grpc.(*Server).Serve(0xc00016d200, {0x232c060, 0xc0001540a0}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:883 +0x611 created by go.etcd.io/etcd/server/v3/embed.(*Etcd).servePeers in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/etcd.go:563 +0x66f goroutine 194 [select, 5 minutes]: github.com/soheilhy/cmux.muxListener.Accept(...) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/cmux.go:262 net/http.(*Server).Serve(0xc00057e000, {0x232c060, 0xc000154100}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/http/server.go:3061 +0x5e8 created by go.etcd.io/etcd/server/v3/embed.(*Etcd).servePeers in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/etcd.go:569 +0xa4d goroutine 195 [IO wait, 5 minutes]: internal/poll.runtime_pollWait(0x7f525c358d30, 0x72) /opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc00023cba0, 0x7f525c205401?, 0x0) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1 internal/poll.(*pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc00023cb80) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:611 +0x425 net.(*netFD).accept(0xc00023cb80) /opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_unix.go:172 +0x3e net.(*TCPListener).accept(0xc000308380) /opt/hostedtoolcache/go/1.21.13/x64/src/net/tcpsock_posix.go:152 +0x3e net.(*TCPListener).Accept(0xc000308380) /opt/hostedtoolcache/go/1.21.13/x64/src/net/tcpsock.go:315 +0x65 go.etcd.io/etcd/client/pkg/v3/transport.(*keepaliveListener).Accept(0xc00044e130) /home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/v3@v3.5.15/transport/keepalive_listener.go:52 +0x53 go.etcd.io/etcd/client/pkg/v3/transport.(*rwTimeoutListener).Accept(0xc0003083a0) /home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/v3@v3.5.15/transport/timeout_listener.go:36 +0x43 github.com/soheilhy/cmux.(*cMux).Serve(0xc000306410) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/cmux.go:170 +0x13c go.etcd.io/etcd/server/v3/embed.(*Etcd).servePeers.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/etcd.go:575 +0x167 go.etcd.io/etcd/server/v3/embed.(*Etcd).servePeers.func3(0xc000308300) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/etcd.go:603 +0x196 created by go.etcd.io/etcd/server/v3/embed.(*Etcd).servePeers in goroutine 162 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/etcd.go:597 +0xe3c goroutine 201 [select]: go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/v3@v3.5.15/fileutil/purge.go:104 +0x4c5 created by go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile in goroutine 185 /home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/v3@v3.5.15/fileutil/purge.go:56 +0x894 goroutine 205 [select, 5 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0005987d0, {0x232fbb8, 0xc000306d20}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:76 +0x1e9 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 162 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:52 +0x20e goroutine 206 [select, 5 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000598800, {0x232fbb8, 0xc000306d70}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:76 +0x1e9 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 162 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:52 +0x20e goroutine 207 [select, 5 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000598830, {0x232fbb8, 0xc000306dc0}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:76 +0x1e9 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 162 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:52 +0x20e goroutine 235 [IO wait, 5 minutes]: internal/poll.runtime_pollWait(0x7f525c358a48, 0x72) /opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc00023dea0, 0xc000ad4000?, 0x0) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1 internal/poll.(*pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00023de80, {0xc000ad4000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405 net.(*netFD).Read(0xc00023de80, {0xc000ad4000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b net.(*conn).Read(0xc000297618, {0xc000ad4000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad bufio.(*Reader).Read(0xc0006ba900, {0xc0001abc40, 0x9, 0x9}) /opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be io.ReadAtLeast({0x23199e0, 0xc0006ba900}, {0xc0001abc40, 0x9, 0x9}, 0x9) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0 io.ReadFull(...) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0001abc40, 0x9, 0x9}, {0x23199e0, 0xc0006ba900}) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:237 +0x9b golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001abc00) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:501 +0xf5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000660240, 0x0?) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:1593 +0x2e5 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 222 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:400 +0x3176 goroutine 236 [select, 5 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0001085a0, 0x1) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:418 +0x1af google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000355960) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:551 +0x14f google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:454 +0x185 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 222 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:452 +0x3aeb goroutine 214 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00011fb00) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:1667 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 208 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:390 +0x3094 goroutine 211 [select]: github.com/tikv/pd/pkg/utils/etcdutil.(*healthChecker).syncer(0xc00023da00, {0x232fbb8, 0xc000306c30}) /home/runner/work/pd/pd/pkg/utils/etcdutil/health_checker.go:94 +0x254 created by github.com/tikv/pd/pkg/utils/etcdutil.initHealthChecker in goroutine 162 /home/runner/work/pd/pd/pkg/utils/etcdutil/health_checker.go:83 +0x3d4 goroutine 212 [select]: github.com/tikv/pd/pkg/utils/etcdutil.(*healthChecker).inspector(0xc00023da00, {0x232fbb8, 0xc000306c30}) /home/runner/work/pd/pd/pkg/utils/etcdutil/health_checker.go:111 +0x245 created by github.com/tikv/pd/pkg/utils/etcdutil.initHealthChecker in goroutine 162 /home/runner/work/pd/pd/pkg/utils/etcdutil/health_checker.go:85 +0x487 goroutine 215 [IO wait]: internal/poll.runtime_pollWait(0x7f525c358b40, 0x72) /opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc00023d9a0, 0xc00068c000?, 0x0) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1 internal/poll.(*pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00023d980, {0xc00068c000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405 net.(*netFD).Read(0xc00023d980, {0xc00068c000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b net.(*conn).Read(0xc000297488, {0xc00068c000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad bufio.(*Reader).Read(0xc00059b260, {0xc00034da80, 0x9, 0x9}) /opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be io.ReadAtLeast({0x23199e0, 0xc00059b260}, {0xc00034da80, 0x9, 0x9}, 0x9) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0 io.ReadFull(...) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc00034da80, 0x9, 0x9}, {0x23199e0, 0xc00059b260}) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:237 +0x9b golang.org/x/net/http2.(*Framer).ReadFrame(0xc00034da40) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:501 +0xf5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00011fb00, 0xc00059b2c0?) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:1593 +0x2e5 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 208 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:400 +0x3176 goroutine 216 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000307720, 0x1) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:418 +0x1af google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000355260) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:551 +0x14f google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:454 +0x185 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 208 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:452 +0x3aeb goroutine 219 [select, 5 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000599a80, {0x232fbb8, 0xc000307db0}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:76 +0x1e9 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 204 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:52 +0x20e goroutine 220 [select, 5 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000599ab0, {0x232fbb8, 0xc000307e00}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:76 +0x1e9 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 204 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:52 +0x20e goroutine 221 [select, 5 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000599ae0, {0x232fbb8, 0xc000307e50}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:76 +0x1e9 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 204 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:52 +0x20e goroutine 243 [select, 5 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0006c1f30, {0x232fbb8, 0xc000109220}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:76 +0x1e9 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 211 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:52 +0x20e goroutine 244 [select, 5 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0006c1f60, {0x232fbb8, 0xc000109270}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:76 +0x1e9 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 211 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:52 +0x20e goroutine 224 [chan receive, 5 minutes]: go.etcd.io/etcd/server/v3/embed.(*serveCtx).registerGateway.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/serve.go:311 +0x65 created by go.etcd.io/etcd/server/v3/embed.(*serveCtx).registerGateway in goroutine 204 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/serve.go:310 +0x32b goroutine 225 [select]: go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.monitorLeader.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/interceptor.go:316 +0x173 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2812 +0x96 created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 204 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/server.go:2810 +0x1cb goroutine 226 [select, 5 minutes]: github.com/soheilhy/cmux.muxListener.Accept(...) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/cmux.go:262 net/http.(*Server).Serve(0xc00057e1e0, {0x232c060, 0xc000155c80}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/http/server.go:3061 +0x5e8 go.etcd.io/etcd/server/v3/embed.(*serveCtx).serve.func3(0x1b48fd0?, {0x232c060, 0xc000155c80}) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/serve.go:175 +0x45 created by go.etcd.io/etcd/server/v3/embed.(*serveCtx).serve in goroutine 204 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/serve.go:174 +0x13e5 goroutine 227 [select, 5 minutes]: github.com/soheilhy/cmux.muxListener.Accept(...) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/cmux.go:262 google.golang.org/grpc.(*Server).Serve(0xc00016d800, {0x232c060, 0xc000155ca0}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:883 +0x611 go.etcd.io/etcd/server/v3/embed.(*serveCtx).serve.func4(0x1b35cb0?, {0x232c060, 0xc000155ca0}) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/serve.go:181 +0x45 created by go.etcd.io/etcd/server/v3/embed.(*serveCtx).serve in goroutine 204 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/embed/serve.go:180 +0x15db goroutine 240 [select, 5 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000108eb0, 0x1) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:418 +0x1af google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004d0850) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:551 +0x14f google.golang.org/grpc/internal/transport.NewServerTransport.func2() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:325 +0x269 created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 239 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:322 +0x260b goroutine 245 [select, 5 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0006c1f90, {0x232fbb8, 0xc0001092c0}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:76 +0x1e9 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 211 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/grpcsync/callback_serializer.go:52 +0x20e goroutine 232 [select, 5 minutes]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006c2820) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:1150 +0x41b created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 230 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:344 +0x268b goroutine 233 [IO wait]: internal/poll.runtime_pollWait(0x7f525c358950, 0x72) /opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000345220, 0xc0006e6000?, 0x0) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1 internal/poll.(*pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000345200, {0xc0006e6000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405 net.(*netFD).Read(0xc000345200, {0xc0006e6000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b net.(*conn).Read(0xc0002975e0, {0xc0006e6000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad github.com/soheilhy/cmux.(*bufferedReader).Read(0xc000355890, {0xc0006e6000, 0x8000, 0x8000}) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/buffer.go:53 +0x318 github.com/soheilhy/cmux.(*MuxConn).Read(0xc000355880, {0xc0006e6000, 0x8000, 0x8000}) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/cmux.go:297 +0x4b bufio.(*Reader).Read(0xc0006ba600, {0xc0001abb60, 0x9, 0x9}) /opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be io.ReadAtLeast({0x23199e0, 0xc0006ba600}, {0xc0001abb60, 0x9, 0x9}, 0x9) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0 io.ReadFull(...) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0001abb60, 0x9, 0x9}, {0x23199e0, 0xc0006ba600}) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:237 +0x9b golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001abb20) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:501 +0xf5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006c2820, {0x232fb80, 0xc0006d0750}, 0x2ee1b00?) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:633 +0x145 google.golang.org/grpc.(*Server).serveStreams(0xc00016d800, {0x232fad8, 0x2ee1b00}, {0x2338300?, 0xc0006c2820}, {0x23376c8?, 0xc000355880}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1021 +0x6bc google.golang.org/grpc.(*Server).handleRawConn.func1() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:957 +0x87 created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 230 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:956 +0x2c7 goroutine 241 [select, 5 minutes]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006c3380) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:1150 +0x41b created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 239 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:344 +0x268b goroutine 242 [IO wait, 5 minutes]: internal/poll.runtime_pollWait(0x7f525c358858, 0x72) /opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0003452a0, 0xc000b24000?, 0x0) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1 internal/poll.(*pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000345280, {0xc000b24000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405 net.(*netFD).Read(0xc000345280, {0xc000b24000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b net.(*conn).Read(0xc0002975f0, {0xc000b24000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad github.com/soheilhy/cmux.(*bufferedReader).Read(0xc0003556d0, {0xc000b24000, 0x8000, 0x8000}) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/buffer.go:53 +0x318 github.com/soheilhy/cmux.(*MuxConn).Read(0xc0003556c0, {0xc000b24000, 0x8000, 0x8000}) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/cmux.go:297 +0x4b bufio.(*Reader).Read(0xc0006bbc80, {0xc0001abee0, 0x9, 0x9}) /opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be io.ReadAtLeast({0x23199e0, 0xc0006bbc80}, {0xc0001abee0, 0x9, 0x9}, 0x9) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0 io.ReadFull(...) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0001abee0, 0x9, 0x9}, {0x23199e0, 0xc0006bbc80}) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:237 +0x9b golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001abea0) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:501 +0xf5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006c3380, {0x232fb80, 0xc000b1e720}, 0x2ee1b00?) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:633 +0x145 google.golang.org/grpc.(*Server).serveStreams(0xc00016d800, {0x232fad8, 0x2ee1b00}, {0x2338300?, 0xc0006c3380}, {0x23376c8?, 0xc0003556c0}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1021 +0x6bc google.golang.org/grpc.(*Server).handleRawConn.func1() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:957 +0x87 created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 239 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:956 +0x2c7 goroutine 264 [select, 5 minutes]: github.com/tikv/pd/pkg/encryption.(*Manager).StartBackgroundLoop(0xc0000c1f80, {0x232fbb8, 0xc000307360}) /home/runner/work/pd/pd/pkg/encryption/key_manager.go:234 +0xc4d created by github.com/tikv/pd/pkg/encryption.TestWatcher in goroutine 162 /home/runner/work/pd/pd/pkg/encryption/key_manager_test.go:342 +0x41a goroutine 271 [select, 5 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc000c2ad00) /home/runner/go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.15/watch.go:545 +0x4d3 created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 264 /home/runner/go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.15/watch.go:292 +0x6cd goroutine 270 [select, 5 minutes]: context.(*cancelCtx).propagateCancel.func2() /opt/hostedtoolcache/go/1.21.13/x64/src/context/context.go:505 +0xe5 created by context.(*cancelCtx).propagateCancel in goroutine 264 /opt/hostedtoolcache/go/1.21.13/x64/src/context/context.go:504 +0x530 goroutine 251 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0006606c0) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:1667 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 246 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:390 +0x3094 goroutine 252 [IO wait]: internal/poll.runtime_pollWait(0x7f525c358760, 0x72) /opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000345fa0, 0xc000b4c000?, 0x0) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1 internal/poll.(*pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000345f80, {0xc000b4c000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405 net.(*netFD).Read(0xc000345f80, {0xc000b4c000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b net.(*conn).Read(0xc000297788, {0xc000b4c000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad bufio.(*Reader).Read(0xc000b3e4e0, {0xc000175000, 0x9, 0x9}) /opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be io.ReadAtLeast({0x23199e0, 0xc000b3e4e0}, {0xc000175000, 0x9, 0x9}, 0x9) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0 io.ReadFull(...) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc000175000, 0x9, 0x9}, {0x23199e0, 0xc000b3e4e0}) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:237 +0x9b golang.org/x/net/http2.(*Framer).ReadFrame(0xc000174fc0) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:501 +0xf5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0006606c0, 0xc000b3e540?) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:1593 +0x2e5 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 246 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:400 +0x3176 goroutine 253 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0001097c0, 0x1) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:418 +0x1af google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004d0af0) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:551 +0x14f google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:454 +0x185 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 246 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_client.go:452 +0x3aeb goroutine 255 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000109950, 0x1) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:418 +0x1af google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004d0b60) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/controlbuf.go:551 +0x14f google.golang.org/grpc/internal/transport.NewServerTransport.func2() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:325 +0x269 created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 254 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:322 +0x260b goroutine 256 [select, 5 minutes]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000b5c340) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:1150 +0x41b created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 254 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:344 +0x268b goroutine 257 [IO wait]: internal/poll.runtime_pollWait(0x7f525c358668, 0x72) /opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0000c0120, 0xc000b5e000?, 0x0) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1 internal/poll.(*pollDesc).waitRead(...) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0000c0100, {0xc000b5e000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405 net.(*netFD).Read(0xc0000c0100, {0xc000b5e000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b net.(*conn).Read(0xc000297778, {0xc000b5e000, 0x8000, 0x8000}) /opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad github.com/soheilhy/cmux.(*bufferedReader).Read(0xc0004d0a90, {0xc000b5e000, 0x8000, 0x8000}) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/buffer.go:53 +0x318 github.com/soheilhy/cmux.(*MuxConn).Read(0xc0004d0a80, {0xc000b5e000, 0x8000, 0x8000}) /home/runner/go/pkg/mod/github.com/soheilhy/cmux@v0.1.5/cmux.go:297 +0x4b bufio.(*Reader).Read(0xc000b3e6c0, {0xc0001750e0, 0x9, 0x9}) /opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be io.ReadAtLeast({0x23199e0, 0xc000b3e6c0}, {0xc0001750e0, 0x9, 0x9}, 0x9) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0 io.ReadFull(...) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0001750e0, 0x9, 0x9}, {0x23199e0, 0xc000b3e6c0}) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:237 +0x9b golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001750a0) /home/runner/go/pkg/mod/golang.org/x/net@v0.25.0/http2/frame.go:501 +0xf5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000b5c340, {0x232fb80, 0xc000b1f8f0}, 0x2ee1b00?) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:633 +0x145 google.golang.org/grpc.(*Server).serveStreams(0xc00016d800, {0x232fad8, 0x2ee1b00}, {0x2338300?, 0xc000b5c340}, {0x23376c8?, 0xc0004d0a80}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1021 +0x6bc google.golang.org/grpc.(*Server).handleRawConn.func1() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:957 +0x87 created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 254 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:956 +0x2c7 goroutine 274 [select, 5 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000c568c0, {0xc000c54ac0, 0x5, 0x5}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:193 +0x13e google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000c568c0, {0xc000c54ac0, 0x5, 0x5}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:173 +0x35b google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000c54a80, {0xc000c54ac0, 0x5, 0x5}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:525 +0x5d io.ReadAtLeast({0x231ebe0, 0xc000c54a80}, {0xc000c54ac0, 0x5, 0x5}, 0x5) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0 io.ReadFull(...) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000c1fd40, {0xc000c54ac0, 0x5, 0x5}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:509 +0x125 google.golang.org/grpc.(*parser).recvMsg(0xc000c54ab0, 0x7fffffff) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/rpc_util.go:614 +0x6e google.golang.org/grpc.recvAndDecompress(0xc000c54ab0, 0xc000c1fd40, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/rpc_util.go:753 +0x93 google.golang.org/grpc.recv(0x0?, {0x7f5201bebb80, 0x2ee1b00}, 0xc000ad1ecc?, {0x0, 0x0}, {0x1fc5d80, 0xc00018f810}, 0x2338300?, 0x0, ...) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/rpc_util.go:833 +0xb7 google.golang.org/grpc.(*csAttempt).recvMsg(0xc000c2aea0, {0x1fc5d80?, 0xc00018f810}, 0x0) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:1085 +0x568 google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000c1fbb4?) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:928 +0x51 google.golang.org/grpc.(*clientStream).withRetry(0xc000c1fb00, 0xc000ad1e18, 0xc000ad1e08) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:761 +0x466 google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c1fb00, {0x1fc5d80?, 0xc00018f810}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:927 +0x1f6 go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc000c508d0) /home/runner/go/pkg/mod/go.etcd.io/etcd/api/v3@v3.5.15/etcdserverpb/rpc.pb.go:6714 +0x66 go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc000c2ad00, {0x2337138, 0xc000c508d0}) /home/runner/go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.15/watch.go:766 +0x67 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 271 /home/runner/go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.15/watch.go:921 +0x9f4 goroutine 273 [select, 5 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:392 +0xde created by google.golang.org/grpc.newClientStreamWithParams in goroutine 271 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:391 +0x1b90 goroutine 275 [select, 5 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc000c2ad00, 0xc00065fad0, 0xc000c34ea0) /home/runner/go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.15/watch.go:812 +0x486 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 271 /home/runner/go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.15/watch.go:562 +0x8eb goroutine 277 [select, 5 minutes]: go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*watchServer).Watch(0xc0001081e0, {0x23370e0?, 0xc000c50f80}) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/watch.go:211 +0x597 go.etcd.io/etcd/api/v3/etcdserverpb._Watch_Watch_Handler({0x1def080?, 0xc0001081e0}, {0x2333b00?, 0xc000c2d818}) /home/runner/go/pkg/mod/go.etcd.io/etcd/api/v3@v3.5.15/etcdserverpb/rpc.pb.go:6743 +0xb6 github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).StreamServerInterceptor.func4({0x1def080, 0xc0001081e0}, {0x2333220?, 0xc00057ee10}, 0xc000c2d7e8, 0x20a9eb0) /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x15f go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainStreamServer.func9.1({0x1def080, 0xc0001081e0}, {0x2333220, 0xc00057ee10}) /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.4.0/chain.go:83 +0xc4 go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.newStreamInterceptor.func1({0x1def080, 0xc0001081e0}, {0x2333220, 0xc00057ee10}, 0xc000c2d7e8, 0xc000c33540) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/interceptor.go:252 +0x6f1 go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainStreamServer.func9({0x1def080, 0xc0001081e0}, {0x2333220, 0xc00057ee10}, 0xc000c2d7e8, 0x20a9eb0) /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.4.0/chain.go:86 +0x219 google.golang.org/grpc.(*Server).processStreamingRPC(0xc00016d800, {0x232fb80, 0xc000c55770}, {0x2338300, 0xc0006c2820}, 0xc000c5e000, 0xc000643890, 0x2e8dd60, 0x0) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1687 +0x1f51 google.golang.org/grpc.(*Server).handleStream(0xc00016d800, {0x2338300, 0xc0006c2820}, 0xc000c5e000) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1801 +0x19f4 google.golang.org/grpc.(*Server).serveStreams.func2.1() /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1027 +0x15d created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 233 /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1038 +0x225 goroutine 278 [select, 5 minutes]: go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*serverWatchStream).sendLoop(0xc0006d5380) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/watch.go:398 +0x3cf go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*watchServer).Watch.func1() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/watch.go:181 +0x2f created by go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*watchServer).Watch in goroutine 277 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/watch.go:180 +0x3e6 goroutine 279 [select, 5 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000c56c80, {0xc000c55810, 0x5, 0x5}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:181 +0x135 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000c56c80, {0xc000c55810, 0x5, 0x5}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:175 +0x3d5 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000c552c0, {0xc000c55810, 0x5, 0x5}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:525 +0x5d io.ReadAtLeast({0x231ebe0, 0xc000c552c0}, {0xc000c55810, 0x5, 0x5}, 0x5) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0 io.ReadFull(...) /opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000c5e000, {0xc000c55810, 0x5, 0x5}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:509 +0x125 google.golang.org/grpc.(*parser).recvMsg(0xc000c55800, 0x200000) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/rpc_util.go:614 +0x6e google.golang.org/grpc.recvAndDecompress(0xc000c55800, 0xc000c5e000, {0x0, 0x0}, 0x200000, 0x0, {0x0, 0x0}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/rpc_util.go:753 +0x93 google.golang.org/grpc.recv(0xa25f79f1ec329ecc?, {0x7f5201be8b98, 0x2ee1b00}, 0xc000b71960?, {0x0, 0x0}, {0x1fb4be0, 0xc000c55b00}, 0xc000b71938?, 0x0, ...) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/rpc_util.go:833 +0xb7 google.golang.org/grpc.(*serverStream).RecvMsg(0xc00057ee10, {0x1fb4be0?, 0xc000c55b00}) /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:1717 +0x311 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredServerStream).RecvMsg(0xc000c2d818, {0x1fb4be0, 0xc000c55b00}) /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:164 +0x53 go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchServer).Recv(0xc000c50f80) /home/runner/go/pkg/mod/go.etcd.io/etcd/api/v3@v3.5.15/etcdserverpb/rpc.pb.go:6762 +0x66 go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*serverWatchStream).recvLoop(0xc0006d5380) /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/watch.go:242 +0x5e go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*watchServer).Watch.func2() /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/watch.go:191 +0x58 created by go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*watchServer).Watch in goroutine 277 /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/watch.go:[190](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:191) +0x4f6 make: *** [Makefile:257: ci-test-job] Error 1 Error: Process completed with exit code 2.
https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516
duplicate with https://github.com/tikv/pd/issues/4879 https://github.com/tikv/pd/issues/6079
meet again https://github.com/tikv/pd/actions/runs/10787696379/job/29916906960?pr=8596
lhy1024
commented
3 weeks ago lhy1024
commented
3 weeks ago 
okJiang
commented
3 days ago
Flaky Test
Which jobs are failing
CI link
https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516
Reason for failure (if possible)
Anything else