vivixx1
commented
3 months ago lora微调时的脚本如下，最大token数设置为8192 export NCCL_P2P_DISABLE=1 export NCCL_IB_DISABLE=1 export NCCL_DEBUG=info export NCCL_SOCKET_IFNAME=eth0 nproc_per_node=6
PYTHONPATH=../../.. \ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 \ torchrun \ --nproc_per_node=$nproc_per_node \ --master_port 29500 \ /swift/examples/pytorch/llm/llm_sft.py \ --model_type qwen1half-72b-chat --model_id_or_path /ai/qwenhalf72bchat/Qwen1.5-72B-Chat/ \ --model_revision master \ --sft_type lora \ --tuner_backend swift \ --template_type AUTO \ --dtype 'bf16' \ --output_dir /ai/outfile/ \ --ddp_backend nccl \ --custom_train_dataset_path /ai/123/训练集4.1knn_41.jsonl \ --train_dataset_sample -1 \ --num_train_epochs 2 \ --max_length 8192 \ --truncation_strategy delete \ --check_dataset_strategy warning \ --lora_rank 8 \ --lora_alpha 32 \ --lora_dropout_p 0.05 \ --lora_target_modules DEFAULT \ --gradient_checkpointing true \ --batch_size 1 \ --weight_decay 0.1 \ --learning_rate 1e-4 \ --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \ --max_grad_norm 0.5 \ --warmup_ratio 0.03 \ --eval_steps 50 \ --save_total_limit 2 \ --logging_steps 1 \ --use_flash_attn true \ --deepspeed default-zero3 \ --save_only_model true \
尝试用增量权重和lora合并权重推理，均会出现RuntimeError: CUDA error: device-side assert triggered，但是在训练的时候不会出现问题 这是增量权重代码 import json import os from transformers.debug_utils import DebugUnderflowOverflow from modelscope import GenerationConfig
os.environ['CUDA_LAUNCH_BLOCKING'] = '1' os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
from swift.llm import ( get_model_tokenizer, get_template, inference, ModelType, get_default_template_type ) from swift.tuners import Swift
ckpt_dir = '/ai/outfile/qwen1half-72b-chat/v6-20240402-111812/checkpoint-106' model_type = ModelType.qwen1half_72b_chat template_type = get_default_template_type(model_type) model_id_or_path = '/ai/qwenhalf72bchat/Qwen1.5-72B-Chat/'
model, tokenizer = get_model_tokenizer(model_type, model_id_or_path=model_id_or_path, model_kwargs={'device_map': 'auto'})
model = Swift.from_pretrained(model, ckpt_dir, inference_mode=True)
debug_overflow = DebugUnderflowOverflow(model) template = get_template(template_type, tokenizer)
model.generation_config = GenerationConfig( max_new_tokens=8192, temperature=0.9, repetition_penalt=1.05, do_sample = True, max_length=512 )
with open('/ai/123/测试集4.2knn.jsonl', 'r', encoding='utf-8') as f: data = [json.loads(item) for item in f]
output_data = [] flag = 0 for line in data:
query = prompt + '题目:' + line['query']
这是合并权重推理代码 import json import os from transformers.debug_utils import DebugUnderflowOverflow from modelscope import GenerationConfig
import os os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
from swift.llm import ( get_model_tokenizer, get_template, inference, ModelType, get_default_template_type )
ckpt_dir = '/ai/outfile/qwen1half-72b-chat/v3-20240331-163441/checkpoint-50-merged/' model_type = ModelType.qwen1half_72b_chat template_type = get_default_template_type(model_type)
model, tokenizer = get_model_tokenizer(model_type, model_kwargs={'device_map': 'auto'}, model_id_or_path=ckpt_dir) debug_overflow = DebugUnderflowOverflow(model) template = get_template(template_type, tokenizer)
with open('/ai/123/测试集4.1knn.jsonl', 'r', encoding='utf-8') as f: data = [json.loads(item) for item in f]
output_data = [] flag = 0 for line in data:
query = prompt + '题目:' + line['query']