Closed xiangking closed 2 years ago
可使用下面方法重写类,下一版本会修复该bug
from ark_nlp.processor.tokenizer.transfomer import TransfomerTokenizer
class TokenTokenizer(TransfomerTokenizer):
"""
Transfomer文本编码器,用于按字符进行分词、ID化、填充等操作
Args:
vocab: transformers词典类对象、词典地址或词典名,用于实现文本分词和ID化
max_seq_len (:obj:`int`): 预设的文本最大长度
""" # noqa: ignore flake8"
def tokenize(self, text, **kwargs):
tokens = []
for token_ in text:
tokenized_token_ = self.vocab.tokenize(token_)
if tokenized_token_ == []:
tokens.extend([token_])
else:
tokens.extend(tokenized_token_)
return tokens
def sequence_to_ids(self, sequence, **kwargs):
return self.sentence_to_ids(sequence, **kwargs)
Environment info
Information