model = AutoPeftModelForCausalLM.from_pretrained("./output", device_map="auto", torch_dtype=torch.float16, trust_remote_code=True)
model = model.quantize(4).cuda()
model.save_pretrained('./quan')
print("quan done")
model = AutoModelForCausalLM.from_pretrained("./quan", device_map="auto", trust_remote_code=True)
报错：
Traceback (most recent call last):
File "/ossfs/node_38300464/workspace/Baichuan2-main/fine-tune/test.py", line 26, in
model = AutoModelForCausalLM.from_pretrained("./quan", device_map="auto", trust_remote_code=True)
File "/root/miniconda3/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 558, in from_pretrained
return model_class.from_pretrained(
File "/root/.cache/huggingface/modules/transformers_modules/quan/modeling_baichuan.py", line 670, in from_pretrained
return super(BaichuanForCausalLM, cls).from_pretrained(pretrained_model_name_or_path, *model_args,
File "/root/miniconda3/lib/python3.10/site-packages/transformers/modeling_utils.py", line 3180, in from_pretrained
) = cls._load_pretrained_model(
File "/root/miniconda3/lib/python3.10/site-packages/transformers/modeling_utils.py", line 3568, in _load_pretrained_model
new_error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model(
File "/root/miniconda3/lib/python3.10/site-packages/transformers/modeling_utils.py", line 745, in _load_state_dict_into_meta_model
set_module_tensor_to_device(model, param_name, param_device, **set_module_kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/accelerate/utils/modeling.py", line 285, in set_module_tensor_to_device
raise ValueError(
ValueError: Trying to set a tensor of shape torch.Size([39321600, 1]) in "weight" (which has shape torch.Size([15360, 5120])), this look incorrect.
LinXin04
commented
8 months ago
model = AutoPeftModelForCausalLM.from_pretrained("./output", device_map="auto", torch_dtype=torch.float16, trust_remote_code=True) model = model.quantize(4).cuda() model.save_pretrained('./quan') print("quan done")
model = AutoModelForCausalLM.from_pretrained("./quan", device_map="auto", trust_remote_code=True)
报错：
Traceback (most recent call last): File "/ossfs/node_38300464/workspace/Baichuan2-main/fine-tune/test.py", line 26, in
model = AutoModelForCausalLM.from_pretrained("./quan", device_map="auto", trust_remote_code=True)
File "/root/miniconda3/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 558, in from_pretrained return model_class.from_pretrained( File "/root/.cache/huggingface/modules/transformers_modules/quan/modeling_baichuan.py", line 670, in from_pretrained return super(BaichuanForCausalLM, cls).from_pretrained(pretrained_model_name_or_path, *model_args, File "/root/miniconda3/lib/python3.10/site-packages/transformers/modeling_utils.py", line 3180, in from_pretrained ) = cls._load_pretrained_model( File "/root/miniconda3/lib/python3.10/site-packages/transformers/modeling_utils.py", line 3568, in _load_pretrained_model new_error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model( File "/root/miniconda3/lib/python3.10/site-packages/transformers/modeling_utils.py", line 745, in _load_state_dict_into_meta_model set_module_tensor_to_device(model, param_name, param_device, **set_module_kwargs) File "/root/miniconda3/lib/python3.10/site-packages/accelerate/utils/modeling.py", line 285, in set_module_tensor_to_device raise ValueError( ValueError: Trying to set a tensor of shape torch.Size([39321600, 1]) in "weight" (which has shape torch.Size([15360, 5120])), this look incorrect.