Open Ariznawlll opened 1 month ago
zhangxu19830126
2f495709d03654899bb9bf149cce6c523b9b738c
- Hardware parameters: - OS type: - Others:
测试中,实际收到的partition-id与当前cn的partition id不一致
log:{"level":"WARN","time":"2024/10/09 06:14:43.113444 +0000","caller":"disttae/txn_table_sharding.go:1019","msg":"receive remote shard request","cn":"63343735-3335-3235-3838-636135363065","table":272514,"partition-id":272515,"local-partitions":[272516],"stack":"github.com/matrixorigin/matrixone/pkg/vm/engine/disttae.(txnTableDelegate).hasAllLocalReplicas\n\t/go/src/github.com/matrixorigin/matrixone/pkg/vm/engine/disttae/txn_table_sharding.go:1024\ngithub.com/matrixorigin/matrixone/pkg/vm/engine/disttae.(txnTableDelegate).isLocalFunc\n\t/go/src/github.com/matrixorigin/matrixone/pkg/vm/engine/disttae/txn_table_sharding.go:1004\ngithub.com/matrixorigin/matrixone/pkg/vm/engine/disttae.(txnTableDelegate).Ranges\n\t/go/src/github.com/matrixorigin/matrixone/pkg/vm/engine/disttae/txn_table_sharding.go:277\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).expandRanges\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:3880\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).generateNodes\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:3985\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compileTableScan\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:1743\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:955\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:976\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:976\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:976\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:1201\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:976\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:1241\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compileQuery\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:823\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compileScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:540\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).Compile\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile2.go:123\ngithub.com/matrixorigin/matrixone/pkg/frontend.createCompile\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/computation_wrapper.go:458\ngithub.com/matrixorigin/matrixone/pkg/frontend.(TxnComputationWrapper).Compile\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/computation_wrapper.go:259\ngithub.com/matrixorigin/matrixone/pkg/frontend.executeStmt\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2664\ngithub.com/matrixorigin/matrixone/pkg/frontend.dispatchStmt\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2567\ngithub.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithIncrStmt\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2536\ngithub.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithWorkspace\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2493\ngithub.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithTxn\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2383\ngithub.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithResponse\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2347\ngithub.com/matrixorigin/matrixone/pkg/frontend.doComQuery\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2954\ngithub.com/matrixorigin/matrixone/pkg/frontend.ExecRequest\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:3029\ngithub.com/matrixorigin/matrixone/pkg/frontend.(Routine).handleRequest\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/routine.go:290\ngithub.com/matrixorigin/matrixone/pkg/frontend.(RoutineManager).Handler\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/routine_manager.go:385\ngithub.com/matrixorigin/matrixone/pkg/frontend.(MOServer).handleRequest\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/server.go:488\ngithub.com/matrixorigin/matrixone/pkg/frontend.(MOServer).handleMessage\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/server.go:456\ngithub.com/matrixorigin/matrixone/pkg/frontend.(MOServer).handleLoop\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/server.go:210\ngithub.com/matrixorigin/matrixone/pkg/frontend.(*MOServer).handleConn\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/server.go:206"}
No response
带sharding的测试结果,启动集群配置中,添加cn和dn的配置,cn的副本数配置为5: cn配置: [cn.shardservice] enable = true dn配置: [dn.shardservice] enable = true max-cn-freeze-time = "5s" max-schedule-tables = 4 git clone https://github.com/matrixorigin/matrixone.git cd mo-sysbench cleanup: sysbench --mysql-host=172.16.22.5 --mysql-port=6001 --mysql-user=dump --mysql-password=111 oltp_update_index.lua --mysql-db=sysbench_db --tables=1 --table_size=100000 --threads=100 --time=30 --report-interval=10 cleanup prepare(prepare前需要先创建db:sysbench_db): sysbench --mysql-host=172.16.22.5 --mysql-port=6001 --mysql-user=dump --mysql-password=111 oltp_update_index.lua --mysql-db=sysbench_db --tables=1 --table_size=100000 --threads=100 --time=30 --report-interval=10 prepare git clone https://github.com/matrixorigin/mo-load.git 根据5个cn,查找cn对应的uuid,以及table_id,将每个cn的范围设置成对应的shard的范围,然后使用5个mo-load工具同时测试tps,将5个tps相加即为总tps,如测试脚本如有需要,联系我 table schema: mysql> show create table sbtest1; +---------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Table | Create Table | +---------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | sbtest1 | CREATE TABLE `sbtest1` ( `id` int NOT NULL AUTO_INCREMENT, `k` int NOT NULL DEFAULT 0, `c` char(120) NOT NULL DEFAULT '', `pad` char(60) NOT NULL DEFAULT '', PRIMARY KEY (`id`) ) partition by range columns (id) (partition p01 values less than (20001), partition p02 values less than (40001), partition p03 values less than (60001), partition p04 values less than (80001), partition p10 values less than (100001)) |
导致这个问题的原因是因为range分区和list分区目前没有支持分区裁剪功能,会扫描所有分区
not working on it
和张旭一起处理,支持range分区裁剪
Is there an existing issue for the same bug?
Branch Name
zhangxu19830126
Commit ID
2f495709d03654899bb9bf149cce6c523b9b738c
Other Environment Information
Actual Behavior
测试中,实际收到的partition-id与当前cn的partition id不一致
log:{"level":"WARN","time":"2024/10/09 06:14:43.113444 +0000","caller":"disttae/txn_table_sharding.go:1019","msg":"receive remote shard request","cn":"63343735-3335-3235-3838-636135363065","table":272514,"partition-id":272515,"local-partitions":[272516],"stack":"github.com/matrixorigin/matrixone/pkg/vm/engine/disttae.(txnTableDelegate).hasAllLocalReplicas\n\t/go/src/github.com/matrixorigin/matrixone/pkg/vm/engine/disttae/txn_table_sharding.go:1024\ngithub.com/matrixorigin/matrixone/pkg/vm/engine/disttae.(txnTableDelegate).isLocalFunc\n\t/go/src/github.com/matrixorigin/matrixone/pkg/vm/engine/disttae/txn_table_sharding.go:1004\ngithub.com/matrixorigin/matrixone/pkg/vm/engine/disttae.(txnTableDelegate).Ranges\n\t/go/src/github.com/matrixorigin/matrixone/pkg/vm/engine/disttae/txn_table_sharding.go:277\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).expandRanges\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:3880\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).generateNodes\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:3985\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compileTableScan\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:1743\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:955\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:976\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:976\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:976\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:1201\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:976\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compilePlanScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:1241\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compileQuery\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:823\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).compileScope\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile.go:540\ngithub.com/matrixorigin/matrixone/pkg/sql/compile.(Compile).Compile\n\t/go/src/github.com/matrixorigin/matrixone/pkg/sql/compile/compile2.go:123\ngithub.com/matrixorigin/matrixone/pkg/frontend.createCompile\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/computation_wrapper.go:458\ngithub.com/matrixorigin/matrixone/pkg/frontend.(TxnComputationWrapper).Compile\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/computation_wrapper.go:259\ngithub.com/matrixorigin/matrixone/pkg/frontend.executeStmt\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2664\ngithub.com/matrixorigin/matrixone/pkg/frontend.dispatchStmt\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2567\ngithub.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithIncrStmt\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2536\ngithub.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithWorkspace\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2493\ngithub.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithTxn\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2383\ngithub.com/matrixorigin/matrixone/pkg/frontend.executeStmtWithResponse\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2347\ngithub.com/matrixorigin/matrixone/pkg/frontend.doComQuery\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:2954\ngithub.com/matrixorigin/matrixone/pkg/frontend.ExecRequest\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/mysql_cmd_executor.go:3029\ngithub.com/matrixorigin/matrixone/pkg/frontend.(Routine).handleRequest\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/routine.go:290\ngithub.com/matrixorigin/matrixone/pkg/frontend.(RoutineManager).Handler\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/routine_manager.go:385\ngithub.com/matrixorigin/matrixone/pkg/frontend.(MOServer).handleRequest\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/server.go:488\ngithub.com/matrixorigin/matrixone/pkg/frontend.(MOServer).handleMessage\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/server.go:456\ngithub.com/matrixorigin/matrixone/pkg/frontend.(MOServer).handleLoop\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/server.go:210\ngithub.com/matrixorigin/matrixone/pkg/frontend.(*MOServer).handleConn\n\t/go/src/github.com/matrixorigin/matrixone/pkg/frontend/server.go:206"}
Expected Behavior
No response
Steps to Reproduce
Additional information
No response