import clip
# downloads model
clip.tokenize(["a"])[0, 1]
from clip.simple_tokenizer import SimpleTokenizer
tokenizer = SimpleTokenizer()
# match whole words
whole_words = {k: v for k, v in tokenizer.encoder.items() if k.endswith("</w>")}
to_trim = len("<w/>")
missed = 0
for token_str, token_int in whole_words.items():
tokenized = tokenizer.encode(token_str[:-to_trim])
if len(tokenized) != 1:
missed += 1
print(f"openai/clip {missed} words out of {len(whole_words)} incorrectly tokenized ({missed/len(whole_words)*100})%")
this prints
openai/clip 1029 words out of 34483 incorrectly tokenized (2.9840791114462197)%
this prints