RuntimeError: Internal: src/sentencepiece_processor.cc(1101) [model_proto->ParseFromArray(serialized.data(), serialized.size())]

[liuyang77886]

环境： Python 3.10.8 cat /proc/version Linux version 5.4.0-99-generic (buildd@lgw01-amd64-007) (gcc version 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04)) #112-Ubuntu SMP Thu Feb 3 13:50:55 UTC 2022 root@autodl-container-28d9119efa-9edf73a5:~/autodl-tmp/Chinese-LangChain# cat /etc/issue Ubuntu 22.04.1 LTS \n \l

我自己下载了模型 /root/autodl-tmp/THUDM/chatglm-6b-int4 /root/autodl-tmp/GanymedeNil/text2vec-large-chinese

做了如下修改：

llm_model_name = '/root/autodl-tmp/THUDM/chatglm-6b-int4'  # 本地模型文件 or huggingface远程仓库
embedding_model_name = '/root/autodl-tmp/GanymedeNil/text2vec-large-chinese'  # 检索模型文件 or huggingface远程仓库

运行： python main.py 报错：

╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮ │ /root/autodl-tmp/Chinese-LangChain/main.py:26 in │ │ │ │ 23 │ │ 24 │ │ 25 config = LangChainCFG() │ │ ❱ 26 application = LangChainApplication(config) │ │ 27 │ │ 28 application.source_service.init_source_vector() │ │ 29 │ │ │ │ /root/autodl-tmp/Chinese-LangChain/clc/langchain_application.py:24 in init │ │ │ │ 21 │ def init(self, config): │ │ 22 │ │ self.config = config │ │ 23 │ │ self.llm_service = ChatGLMService() │ │ ❱ 24 │ │ self.llm_service.load_model(model_name_or_path=self.config.llm_model_name) │ │ 25 │ │ # self.llm_service.load_model_on_gpus(model_name_or_path=self.config.llm_model_n │ │ 26 │ │ self.source_service = SourceService(config) │ │ 27 │ │ │ │ /root/autodl-tmp/Chinese-LangChain/clc/gpt_service.py:55 in load_model │ │ │ │ 52 │ │ │ 53 │ def load_model(self, │ │ 54 │ │ │ │ model_name_or_path: str = "THUDM/chatglm-6b"): │ │ ❱ 55 │ │ self.tokenizer = AutoTokenizer.from_pretrained( │ │ 56 │ │ │ model_name_or_path, │ │ 57 │ │ │ trust_remote_code=True │ │ 58 │ │ ) │ │ │ │ /root/miniconda3/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py:693 │ │ in from_pretrained │ │ │ │ 690 │ │ │ │ raise ValueError( │ │ 691 │ │ │ │ │ f"Tokenizer class {tokenizer_class_candidate} does not exist or is n │ │ 692 │ │ │ │ ) │ │ ❱ 693 │ │ │ return tokenizer_class.from_pretrained(pretrained_model_name_or_path, input │ │ 694 │ │ │ │ 695 │ │ # Otherwise we have to be creative. │ │ 696 │ │ # if model is an encoder decoder, the encoder tokenizer class is used by default │ │ │ │ /root/miniconda3/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1812 in │ │ from_pretrained │ │ │ │ 1809 │ │ │ else: │ │ 1810 │ │ │ │ logger.info(f"loading file {file_path} from cache at {resolved_vocab_fil │ │ 1811 │ │ │ │ ❱ 1812 │ │ return cls._from_pretrained( │ │ 1813 │ │ │ resolved_vocab_files, │ │ 1814 │ │ │ pretrained_model_name_or_path, │ │ 1815 │ │ │ init_configuration, │ │ │ │ /root/miniconda3/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1975 in │ │ _from_pretrained │ │ │ │ 1972 │ │ │ │ 1973 │ │ # Instantiate tokenizer. │ │ 1974 │ │ try: │ │ ❱ 1975 │ │ │ tokenizer = cls(init_inputs, **init_kwargs) │ │ 1976 │ │ except OSError: │ │ 1977 │ │ │ raise OSError( │ │ 1978 │ │ │ │ "Unable to load vocabulary from file. " │ │ │ │ /root/.cache/huggingface/modules/transformers_modules/chatglm-6b-int4/tokenization_chatglm.py:22 │ │ 1 in init │ │ │ │ 218 │ │ self.mask_token = mask_token │ │ 219 │ │ self.gmask_token = gmask_token │ │ 220 │ │ │ │ ❱ 221 │ │ self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens) │ │ 222 │ │ │ │ 223 │ │ """ Initialisation """ │ │ 224 │ │ │ │ /root/.cache/huggingface/modules/transformers_modules/chatglm-6b-int4/tokenization_chatglm.py:64 │ │ in init │ │ │ │ 61 │ │ self.special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "", "", "< │ │ 62 │ │ self.max_blank_length = max_blank_length │ │ 63 │ │ self.byte_fallback = byte_fallback │ │ ❱ 64 │ │ self.text_tokenizer = TextTokenizer(vocab_file) │ │ 65 │ │ │ 66 │ def _get_text_tokenizer(self): │ │ 67 │ │ return self.text_tokenizer │ │ │ │ /root/.cache/huggingface/modules/transformers_modules/chatglm-6b-int4/tokenization_chatglm.py:22 │ │ in init │ │ │ │ 19 class TextTokenizer: │ │ 20 │ def init(self, model_path): │ │ 21 │ │ self.sp = spm.SentencePieceProcessor() │ │ ❱ 22 │ │ self.sp.Load(model_path) │ │ 23 │ │ self.num_tokens = self.sp.vocab_size() │ │ 24 │ │ │ 25 │ def encode(self, text): │ │ │ │ /root/miniconda3/lib/python3.10/site-packages/sentencepiece/init.py:905 in Load │ │ │ │ 902 │ │ raise RuntimeError('model_file and model_proto must be exclusive.') │ │ 903 │ if model_proto: │ │ 904 │ │ return self.LoadFromSerializedProto(model_proto) │ │ ❱ 905 │ return self.LoadFromFile(model_file) │ │ 906 │ │ 907 │ │ 908 # Register SentencePieceProcessor in _sentencepiece: │ │ │ │ /root/miniconda3/lib/python3.10/site-packages/sentencepiece/init.py:310 in LoadFromFile │ │ │ │ 307 │ │ return _sentencepiece.SentencePieceProcessor_serialized_model_proto(self) │ │ 308 │ │ │ 309 │ def LoadFromFile(self, arg): │ │ ❱ 310 │ │ return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg) │ │ 311 │ │ │ 312 │ def _EncodeAsIds(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, r │ │ 313 │ │ return _sentencepiece.SentencePieceProcessor__EncodeAsIds(self, text, enable_sam │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ RuntimeError: Internal: src/sentencepiece_processor.cc(1101) [model_proto->ParseFromArray(serialized.data(), serialized.size())]