baleian
commented
7 years ago class Bracket(list): def init(self, arr): list.init(self, arr)
def tokenize_for_word(word):
word = word.replace('(', ' ( ').replace(')', ' ) ')
tokens = word.split()
while True:
left = len(tokens) - 1 - tokens[::-1].index('(') if '(' in tokens else None
right = tokens[(left if left is not None else 0):].index(')') +left if ')' in tokens else None
if left is None and right is None:
break
elif left is None and right is not None:
tokens[right:right+1] = []
elif left is not None and right is None:
tokens[left:left+1] = []
else:
tokens[left:right+1] = [tokens[left+1:right]]
set_bracket(tokens)
return tokensdef set_bracket(tokens): for i in range(len(tokens)): if type(tokens[i]) is list: set_bracket(tokens[i]) tokens[i] = Bracket(tokens[i])
class Variation(list): def init(self, arr): list.init(self, arr)
def set_word_variation_dictionary(tokens, word_variation_dic): for token in tokens: if type(token) is Bracket: set_word_variation_dictionary(token, word_variation_dic) elif token not in word_variation_dic: word_variation_dic[token] = Variation([(token, 1)])
def link_tokens_to_word_variation_dic(tokens, word_variation_dic): for idx, token in enumerate(tokens): if type(token) is Bracket: link_tokens_to_word_variation_dic(token, word_variation_dic) else: tokens[idx] = word_variation_dic[token]
variation_words = [ [('수퍼', 1), ('슈퍼', 1)], [('CU', 1), ('씨유', 0.5), ('훼미리마트', 0.3)], [('GS', 1), ('지에스', 0.5)], [('25', 1), ('이십오', 0.1)], [('365', 1), ('삼육오', 0.5)] ]
def is_word_includevword(word, vwords): for vword, in vwords: if vword in word: return vword return False
def get_word_variations(word, rate=1):
variations = [(word, rate)]
for vwords in variation_words:
size = len(variations)
for i in range(size):
w, r = variations[i]
_w = is_word_include_vword(w, vwords)
if not _w: continue
for vword, vrate in vwords:
if vword is _w: continue
variations.append((w.replace(_w, vword), r * vrate))
return variations
variation_charactors = { 'A': '에이', 'B': '비', 'C': '씨', 'D': '디', 'E': '이', 'F': '에프', 'G': '지', 'H': '에이치', 'I': '아이', 'J': '제이', 'K': '케이', 'L': '엘', 'M': '엠', 'N': '엔', 'O': '오', 'P': '피', 'Q': '큐', 'R': '알', 'S': '에스', 'T': '티', 'U': '유', 'V': '브이', 'W': '더블유', 'X': '엑스', 'Y': '와이', 'Z': '지', '1': '일', '2': '이', '3': '삼', '4': '사', '5': '오', '6': '육', '7': '칠', '8': '팔', '9': '구', '0': '공' }
def get_charactor_variations(word, rate=1): variations = [(word, rate)] for ch in variation_charactors: size = len(variations) for i in range(size): w, r = variations[i] if ch not in w: continue variations.append((w.replace(ch, variation_charactors[ch]), 0)) return variations
import itertools
class Permutation(list): def init(self, arr): list.init(self, arr)
def make_bracket_permutation(tokens): for idx, token in enumerate(tokens): if type(token) is Bracket: tokens[idx] = make_bracket_permutation(token)
ret = []
perm = itertools.permutations(tokens)
for p in perm:
tmp = [Bracket(list(p))]
for i in range(len(tokens)):
tmp2 = []
for t in tmp:
if type(t[i]) is Permutation:
for ti in t[i]:
ll = t[:]
ll[i:i+1] = [ti]
tmp2.append(Bracket(ll))
else:
tmp2.append(t)
tmp = tmp2
ret += tmp
return Permutation(ret)def make_validation_str(perm):
ret = [([], 1)]
for p in perm:
size = len(ret)
if type(p) is Variation:
tmp = []
for i in range(size):
for q in p:
tmp.append((ret[i][0] + q[0], ret[i][1] q[1]))
ret = tmp
elif type(p) is Bracket:
tmp = []
for i in range(size):
qs = make_validation_str(p)
for q in qs:
tmp.append((ret[i][0] + q[0], ret[i][1] q[1]))
ret += tmp
return ret
import unicodedata
def check_hangul(ch): valid_type = ['Lo'] if unicodedata.category(ch) in valid_type: unicode_names = unicodedata.name(ch).split() if 'HANGUL' in unicode_names and 'SYLLABLE' in unicode_names: return True else: return False else: return False
def conv_compatibility_jamo(ch): unicode_names = unicodedata.name(ch)
def conv_jaso(ch): if not check_hangul(ch): return [ch]
def str_to_jaso(str): ret = [] for ch in str: ret += conv_jaso(ch) return ret