search

matrixorigin / matrixone

Hyperconverged cloud-edge native database
https://docs.matrixorigin.cn/en
Apache License 2.0
1.78k stars 275 forks source link

[Bug]: data race in UT: TestKill #19245

Open ouyuanning opened 1 week ago

ouyuanning commented 1 week ago

Is there an existing issue for the same bug?

Branch Name

1.2-dev

Commit ID

1.2-dev

Other Environment Information

- Hardware parameters:
- OS type:
- Others:

Actual Behavior

https://github.com/matrixorigin/matrixone/actions/runs/11288345470/job/31395934322?pr=19240


2024-10-11T07:54:51.7401677Z ==================
2024-10-11T07:54:51.7401872Z WARNING: DATA RACE
2024-10-11T07:54:51.7402126Z Read at 0x00c00321a400 by goroutine 774:
2024-10-11T07:54:51.7402795Z   github.com/matrixorigin/matrixone/pkg/frontend.(*Routine).killConnection.func1()
2024-10-11T07:54:51.7403522Z       /home/runner/work/matrixone/matrixone/pkg/frontend/routine.go:400 +0x30
2024-10-11T07:54:51.7404266Z   github.com/matrixorigin/matrixone/pkg/frontend.(*Routine).execCallbackBasedOnRequest()
2024-10-11T07:54:51.7404986Z       /home/runner/work/matrixone/matrixone/pkg/frontend/routine.go:127 +0xbb
2024-10-11T07:54:51.7405588Z   github.com/matrixorigin/matrixone/pkg/frontend.(*Routine).killConnection()
2024-10-11T07:54:51.7406299Z       /home/runner/work/matrixone/matrixone/pkg/frontend/routine.go:406 +0x92
2024-10-11T07:54:51.7406873Z   github.com/matrixorigin/matrixone/pkg/frontend.(*RoutineManager).kill()
2024-10-11T07:54:51.7407686Z       /home/runner/work/matrixone/matrixone/pkg/frontend/routine_manager.go:345 +0x15d
2024-10-11T07:54:51.7408112Z   github.com/matrixorigin/matrixone/pkg/frontend.doKill()
2024-10-11T07:54:51.7408967Z       /home/runner/work/matrixone/matrixone/pkg/frontend/mysql_cmd_executor.go:1366 +0x139
2024-10-11T07:54:51.7409426Z   github.com/matrixorigin/matrixone/pkg/frontend.handleKill()
2024-10-11T07:54:51.7410295Z       /home/runner/work/matrixone/matrixone/pkg/frontend/mysql_cmd_executor.go:1375 +0x2be4
2024-10-11T07:54:51.7466921Z   github.com/matrixorigin/matrixone/pkg/frontend.execInFrontend()
2024-10-11T07:54:51.7467762Z       /home/runner/work/matrixone/matrixone/pkg/frontend/self_handle.go:354 +0x2be5
2024-10-11T07:54:51.7468242Z   github.com/matrixorigin/matrixone/pkg/frontend.executeStmt()
2024-10-11T07:54:51.7469135Z       /home/runner/work/matrixone/matrixone/pkg/frontend/mysql_cmd_executor.go:2491 +0x1612
2024-10-11T07:54:51.7469612Z   github.com/matrixorigin/matrixone/pkg/frontend.dispatchStmt()
2024-10-11T07:54:51.7470478Z       /home/runner/work/matrixone/matrixone/pkg/frontend/mysql_cmd_executor.go:2459 +0x498
2024-10-11T07:54:51.7471077Z   github.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithIncrStmt()
2024-10-11T07:54:51.7471929Z       /home/runner/work/matrixone/matrixone/pkg/frontend/mysql_cmd_executor.go:2428 +0x308
2024-10-11T07:54:51.7472539Z   github.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithWorkspace()
2024-10-11T07:54:51.7473790Z       /home/runner/work/matrixone/matrixone/pkg/frontend/mysql_cmd_executor.go:2388 +0x834
2024-10-11T07:54:51.7474347Z   github.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithTxn()
2024-10-11T07:54:51.7475210Z       /home/runner/work/matrixone/matrixone/pkg/frontend/mysql_cmd_executor.go:2280 +0x1f9
2024-10-11T07:54:51.7475806Z   github.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithResponse()
2024-10-11T07:54:51.7476657Z       /home/runner/work/matrixone/matrixone/pkg/frontend/mysql_cmd_executor.go:2244 +0x5c4
2024-10-11T07:54:51.7477135Z   github.com/matrixorigin/matrixone/pkg/frontend.doComQuery()
2024-10-11T07:54:51.7478021Z       /home/runner/work/matrixone/matrixone/pkg/frontend/mysql_cmd_executor.go:2837 +0x35ed
2024-10-11T07:54:51.7478501Z   github.com/matrixorigin/matrixone/pkg/frontend.ExecRequest()
2024-10-11T07:54:51.7479350Z       /home/runner/work/matrixone/matrixone/pkg/frontend/mysql_cmd_executor.go:2912 +0xd91
2024-10-11T07:54:51.7479943Z   github.com/matrixorigin/matrixone/pkg/frontend.(*Routine).handleRequest()
2024-10-11T07:54:51.7480681Z       /home/runner/work/matrixone/matrixone/pkg/frontend/routine.go:291 +0xa39
2024-10-11T07:54:51.7481280Z   github.com/matrixorigin/matrixone/pkg/frontend.(*RoutineManager).Handler()
2024-10-11T07:54:51.7482107Z       /home/runner/work/matrixone/matrixone/pkg/frontend/routine_manager.go:545 +0x13f3
2024-10-11T07:54:51.7482896Z   github.com/matrixorigin/matrixone/pkg/frontend.(*RoutineManager).Handler-fm()
2024-10-11T07:54:51.7483169Z       <autogenerated>:1 +0x6d
2024-10-11T07:54:51.7483572Z   github.com/fagongzi/goetty/v2.(*server).doConnection()
2024-10-11T07:54:51.7485030Z       /home/runner/go/pkg/mod/github.com/matrixorigin/goetty/v2@v2.0.0-20240611082008-a4de209fff3d/application.go:393 +0x426
2024-10-11T07:54:51.7485552Z   github.com/fagongzi/goetty/v2.(*server).doConnection-fm()
2024-10-11T07:54:51.7534614Z       <autogenerated>:1 +0x47
2024-10-11T07:54:51.7535150Z   github.com/fagongzi/goetty/v2.(*server).doStart.func2.1()
2024-10-11T07:54:51.7536666Z       /home/runner/go/pkg/mod/github.com/matrixorigin/goetty/v2@v2.0.0-20240611082008-a4de209fff3d/application.go:356 +0x106
2024-10-11T07:54:51.7536689Z 
2024-10-11T07:54:51.7537013Z Previous write at 0x00c00321a400 by goroutine 770:
2024-10-11T07:54:51.7537864Z   github.com/matrixorigin/matrixone/pkg/frontend.(*RoutineManager).Closed.(*Routine).cleanup.func4()
2024-10-11T07:54:51.7538610Z       /home/runner/work/matrixone/matrixone/pkg/frontend/routine.go:448 +0x346
2024-10-11T07:54:51.7538826Z   sync.(*Once).doSlow()
2024-10-11T07:54:51.7539395Z       /opt/hostedtoolcache/go/1.22.3/x64/src/sync/once.go:74 +0xf0
2024-10-11T07:54:51.7539595Z   sync.(*Once).Do()
2024-10-11T07:54:51.7540136Z       /opt/hostedtoolcache/go/1.22.3/x64/src/sync/once.go:65 +0x44
2024-10-11T07:54:51.7540673Z   github.com/matrixorigin/matrixone/pkg/frontend.(*Routine).cleanup()
2024-10-11T07:54:51.7541400Z       /home/runner/work/matrixone/matrixone/pkg/frontend/routine.go:416 +0x450
2024-10-11T07:54:51.7541986Z   github.com/matrixorigin/matrixone/pkg/frontend.(*RoutineManager).Closed()
2024-10-11T07:54:51.7542805Z       /home/runner/work/matrixone/matrixone/pkg/frontend/routine_manager.go:329 +0x3f9
2024-10-11T07:54:51.7543146Z   github.com/fagongzi/goetty/v2.(*baseIO).Close()
2024-10-11T07:54:51.7544542Z       /home/runner/go/pkg/mod/github.com/matrixorigin/goetty/v2@v2.0.0-20240611082008-a4de209fff3d/

Expected Behavior

No response

Steps to Reproduce

https://github.com/matrixorigin/matrixone/actions/runs/11288345470/job/31395934322?pr=19240

Additional information

No response

ouyuanning commented 1 week ago

log: job-logs_副本.txt

daviszhen commented 1 week ago

main 上已经解决。升级到main 上就没有问题了。

aylei commented 2 days ago

main 上已经解决。升级到main 上就没有问题了。

still appears in UT: https://github.com/matrixorigin/matrixone/actions/runs/11377639018/job/31652127205?pr=19363

2024-10-17T03:23:45.1413991Z {"Time":"2024-10-17T03:23:45.140954276Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"WARNING: DATA RACE\n"}
2024-10-17T03:23:45.1417270Z {"Time":"2024-10-17T03:23:45.141055435Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"Read at 0x00c00605a580 by goroutine 1020:\n"}
2024-10-17T03:23:45.1420896Z {"Time":"2024-10-17T03:23:45.141085742Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"  github.com/matrixorigin/matrixone/pkg/frontend.(*Routine).killConnection.func1()\n"}
2024-10-17T03:23:45.1425227Z {"Time":"2024-10-17T03:23:45.141092255Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"      /home/runner/work/matrixone/matrixone/pkg/frontend/routine.go:396 +0x30\n"}
2024-10-17T03:23:45.1429618Z {"Time":"2024-10-17T03:23:45.141125046Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"  github.com/matrixorigin/matrixone/pkg/frontend.(*Routine).execCallbackBasedOnRequest()\n"}
2024-10-17T03:23:45.1433583Z {"Time":"2024-10-17T03:23:45.141131408Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"      /home/runner/work/matrixone/matrixone/pkg/frontend/routine.go:126 +0xbb\n"}
2024-10-17T03:23:45.1437439Z {"Time":"2024-10-17T03:23:45.141161925Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"  github.com/matrixorigin/matrixone/pkg/frontend.(*Routine).killConnection()\n"}
2024-10-17T03:23:45.1441254Z {"Time":"2024-10-17T03:23:45.141168016Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"      /home/runner/work/matrixone/matrixone/pkg/frontend/routine.go:402 +0x92\n"}
2024-10-17T03:23:45.1453168Z {"Time":"2024-10-17T03:23:45.145039616Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"  github.com/matrixorigin/matrixone/pkg/frontend.(*RoutineManager).kill()\n"}
2024-10-17T03:23:45.1457255Z {"Time":"2024-10-17T03:23:45.145049915Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"      /home/runner/work/matrixone/matrixone/pkg/frontend/routine_manager.go:341 +0x15d\n"}
2024-10-17T03:23:45.1472167Z {"Time":"2024-10-17T03:23:45.146361792Z","Action":"output","Package":"github.com/matrixorigin/matrixone/pkg/frontend","Test":"TestKill","Output":"  github.com/matrixorigin/matrixone/pkg/frontend.doKill()\n"}