Closed loubnabnl closed 5 months ago
fixes https://github.com/bigcode-project/bigcode-evaluation-harness/issues/182 ChatGLM3-6B requires left padding which wasn't supported in the harness, you need to add --left_padding flag to force it.
--left_padding
accelerate launch main.py \ --model THUDM/chatglm3-6b \ --tasks humaneval \ --n_samples 1 \ --batch_size 1 \ --allow_code_execution \ --use_auth_token \ --do_sample False \ --precision fp16 \ --trust_remote_code \ --save_generations \ --left_padding
output
{ "humaneval": { "pass@1": 0.29878048780487804 }, "config": { "prefix": "", "do_sample": false, "temperature": 0.2, "top_k": 0, "top_p": 0.95, "n_samples": 1, "eos": "<|endoftext|>", "seed": 0, "model": "THUDM/chatglm3-6b", "modeltype": "causal", "peft_model": null, "revision": null, "use_auth_token": true, "trust_remote_code": true, "tasks": "humaneval", "instruction_tokens": null, "batch_size": 1, "max_length_generation": 512, "precision": "fp16", "load_in_8bit": false, "load_in_4bit": false, "left_padding": true, "limit": null, "limit_start": 0, "save_every_k_tasks": -1, "postprocess": true, "allow_code_execution": true, "generation_only": false, "load_generations_path": null, "load_data_path": null, "metric_output_path": "evaluation_results.json", "save_generations": true, "load_generations_intermediate_paths": null, "save_generations_path": "generations.json", "save_references": false, "save_references_path": "references.json", "prompt": "prompt", "max_memory_per_gpu": null, "check_references": false } }
fixes https://github.com/bigcode-project/bigcode-evaluation-harness/issues/182 ChatGLM3-6B requires left padding which wasn't supported in the harness, you need to add
--left_padding
flag to force it.output