Open guoqingbao opened 10 months ago
I have managed to resolve the problem by revising the following code in "export" function, refer to PR #18
def serialize(k):
# w = None
# if isinstance(k, torch.Tensor):
# w = k
# elif "GeneralQuantLinear" in str(k.__class__) and EXPAND:
# w = k.build()[0].T
# elif "GeneralQuantLinear" not in str(k.__class__):
# w = k.weight
if hasattr(k, "qweight"):
for w in [k.qweight.type(torch.int32), k.qzeros.type(torch.int32), k.scales.type(torch.float32)]:
print("Quant")
print(w.shape)
t = w.T.contiguous().view(-1).detach().cpu().numpy()
f.write(memoryview(t))
else:
if hasattr(k, "weight"):
w = k.weight
else:
w = k
print("Regular")
print(w.shape)
t = w.contiguous().view(-1).detach().cpu().type(torch.float32).numpy()
f.write(memoryview(t))
# del state_dict[key]
# first write out the header
p['n_heads'] = model.layers[0].self_attn.num_heads
# hidden_dim = model.layers[0].mlp.up_proj.build()[0].shape[1]
# p['dim'] = model.layers[0].mlp.up_proj.build()[0].shape[0]
hidden_dim = model.layers[0].mlp.up_proj.out_features
p['dim'] = model.layers[0].mlp.up_proj.in_features
I have the following errors:
My Python package versions:
Package Version
auto-gptq 0.3.1 huggingface-hub 0.16.4 torch 2.1.0a0+fe05266 transformer-engine 0.7.0 transformers 4.31.0