baichuan2-13b run.py Assertion failed: mpiSize == tp * pp

[PaulX1029]

System Info

nvidia/cuda:12.1.0-devel-ubuntu22.04

Who can help?

No response

Information

• [X] The official example scripts
• [X] My own modified scripts

Tasks

• [X] An officially supported task in the examples folder (such as GLUE/SQuAD, ...)
• [ ] My own task or dataset (give details below)

Reproduction

convert_ckpts: python convert_checkpoint.py --model_version v2_13b --model_dir /data/checkpoint/BaiChuan2-13B/ --dtype float16 --output_dir /home/TensorRT-LLM/data/BaiChuan2-13B/trt_ckpts/fp16/2-gpu --world_size2 --tp_size 2 build: trtllm-build --checkpoint_dir /home/TensorRT-LLM/data/BaiChuan2-13B/trt_ckpts/fp16/2-gpu-tp2/ --output_dir /home/TensorRT-LLM/data/BaiChuan2-13B/trt_engines/fp16/2-gpu-tp2/ --gemm_plugin float16 --max_batch_size=32 --max_input_len=2048 --max_output_len=2048 --tp_size 2 custom run: CUDA_VISIBLE_DEVICES=0,1 mpirun -n 2 python3 /home/TensorRT-LLM/examples/run_benchmark.py --engine_dir ./BaiChuan2-13B/trt_engines/fp16/2-gpu-tp2/ --tokenizer_dir /data/checkpoint/BaiChuan2-13B/ --max_output_len 1024 --single_input_tokenid_length 2048 --benchmark_model_name BaiChuan2-13B --input_file ./BaiChuan2-13B/speed_data/test/input_ids_bsz32_2048.csv --run_profiling

Expected behavior

正常使用run.py脚本

actual behavior

error

additional notes

{ "version": "0.9.0.dev2024031900", "pretrained_config": { "architecture": "BaichuanForCausalLM", "dtype": "float16", "logits_dtype": "float32", "vocab_size": 125696, "max_position_embeddings": 4096, "hidden_size": 5120, "num_hidden_layers": 40, "num_attention_heads": 40, "num_key_value_heads": 40, "head_size": 128, "hidden_act": "silu", "intermediate_size": 13696, "norm_epsilon": 1e-06, "position_embedding_type": "alibi", "use_parallel_embedding": false, "embedding_sharding_dim": 0, "share_embedding_table": false, "mapping": { "world_size": 2, "tp_size": 2, "pp_size": 1 }, "quantization": { "quant_algo": null, "kv_cache_quant_algo": null, "group_size": 128, "has_zero_point": false, "pre_quant_scale": false, "exclude_modules": null }, "kv_dtype": "float16", "use_prompt_tuning": false }, "build_config": { "max_input_len": 2048, "max_output_len": 2048, "max_batch_size": 32, "max_beam_width": 1, "max_num_tokens": 65536, "max_prompt_embedding_table_size": 0, "gather_context_logits": false, "gather_generation_logits": false, "strongly_typed": false, "builder_opt": null, "profiling_verbosity": "layer_names_only", "enable_debug_output": false, "max_draft_len": 0, "use_refit": false, "input_timing_cache": null, "output_timing_cache": "model.cache", "lora_config": { "lora_dir": [], "lora_ckpt_source": "hf", "max_lora_rank": 64, "lora_target_modules": [], "trtllm_modules_to_hf_modules": {} }, "auto_parallel_config": { "world_size": 1, "gpus_per_node": 8, "cluster_key": "A100-SXM-80GB", "cluster_info": null, "sharding_cost_model": "alpha_beta", "comm_cost_model": "alpha_beta", "enable_pipeline_parallelism": false, "enable_shard_unbalanced_shape": false, "enable_shard_dynamic_shape": false, "enable_reduce_scatter": true, "builder_flags": null, "debug_mode": false, "infer_shape": true, "validation_mode": false, "same_buffer_io": { "past_keyvalue(\d+)": "present_keyvalue\1" }, "same_spec_io": {}, "sharded_io_allowlist": [ "past_keyvalue\d+", "present_keyvalue\d*" ], "fast_reduce": true, "fill_weights": false, "parallel_config_cache": null, "profile_cache": null, "dump_path": null, "debug_outputs": [] }, "plugin_config": { "bert_attention_plugin": "float16", "gpt_attention_plugin": "float16", "gemm_plugin": "float16", "smooth_quant_gemm_plugin": null, "identity_plugin": null, "layernorm_quantization_plugin": null, "rmsnorm_quantization_plugin": null, "nccl_plugin": "float16", "lookup_plugin": null, "lora_plugin": null, "weight_only_groupwise_quant_matmul_plugin": null, "weight_only_quant_matmul_plugin": null, "quantize_per_token_plugin": false, "quantize_tensor_plugin": false, "moe_plugin": "float16", "context_fmha": true, "context_fmha_fp32_acc": false, "paged_kv_cache": true, "remove_input_padding": true, "use_custom_all_reduce": true, "multi_block_mode": false, "enable_xqa": true, "attention_qk_half_accumulation": false, "tokens_per_block": 128, "use_paged_context_fmha": false, "use_context_fmha_for_generation": false, "dense_context_fmha": false, "pos_shift": false, "multiple_profiles": false } } }

[byshiue]

Could you share

1. What branch do you use? If you don't use latest main branch, we suggest taking a try.
2. What are the full error messages?

[PaulX1029]

@byshiue it’s my fault，mpirun version wrong

[dingjingzhen]

@byshiue it’s my fault，mpirun version wrong

I have the same problem, what version do you use