emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F" # emoticons
u"\U0001F300-\U0001F5FF" # symbols & pictographs
u"\U0001F680-\U0001F6FF" # transport & map symbols
u"\U0001F1E0-\U0001F1FF" # flags (iOS)
"]+", flags=re.UNICODE)re.sub('\s'," ", doc) doc.lower()
Last updated
Was this helpful?
Was this helpful?
re.sub('[-=+,#/\?:^$.@*\"※~&%ㆍ!』\\‘|\(\)\[\]\<\>`\'…》]', ' ', doc)re.compile('[^ ㄱ-ㅣ가-힣|a-z]+').sub('', doc)def konlpy_morp_analy(text):
#print("docs_length:",len(text))
try:
result = komoran.pos(text)
return result
except:
return []
def konlpy_parser(m_list):
tokens = []
df = pd.DataFrame(columns = ["text", "type"])
for data,m_type in m_list:
if is_valid(data, m_type):
tokens += [data]
return tokensh_s_set = h_stopwords
if len(data) <= 1 or data in h_s_set:
return Falsee_s_set = set(stopwords.words('english'))
if m_type == "SL" and (data in e_s_set or len(data) <= 2):
return Falseif data in stop_set:
return Falseif any(i in single_stop for i in data):
return Falseif m_type not in ["SH", "SL"] and
data != re.compile('[^ ㄱ-ㅣ가-힣|a-z]+').sub('', data):
return Falseif len(data) >= 15:
return Falsesign_list = sign_dict.values()
if m_type in sign_list:
return True