77 lines
1.9 KiB
Python
77 lines
1.9 KiB
Python
#import hazm
|
|
from cleantext import clean
|
|
import re
|
|
|
|
def cleanhtml(raw_html):
|
|
cleanr = re.compile('<.*?>')
|
|
cleantext = re.sub(cleanr, '', raw_html)
|
|
return cleantext
|
|
|
|
#normalizer = hazm.Normalizer()
|
|
wierd_pattern = re.compile("["
|
|
u"\U0001F600-\U0001F64F" # emoticons
|
|
u"\U0001F300-\U0001F5FF" # symbols & pictographs
|
|
u"\U0001F680-\U0001F6FF" # transport & map symbols
|
|
u"\U0001F1E0-\U0001F1FF" # flags (iOS)
|
|
u"\U00002702-\U000027B0"
|
|
u"\U000024C2-\U0001F251"
|
|
u"\U0001f926-\U0001f937"
|
|
u'\U00010000-\U0010ffff'
|
|
u"\u200d"
|
|
u"\u2640-\u2642"
|
|
u"\u2600-\u2B55"
|
|
u"\u23cf"
|
|
u"\u23e9"
|
|
u"\u231a"
|
|
u"\u3030"
|
|
u"\ufe0f"
|
|
u"\u2069"
|
|
u"\u2066"
|
|
# u"\u200c"
|
|
u"\u2068"
|
|
u"\u2067"
|
|
"]+", flags=re.UNICODE)
|
|
|
|
def cleaning(text):
|
|
text = text.strip()
|
|
|
|
# regular cleaning
|
|
# text = clean(text,
|
|
# fix_unicode=True,
|
|
# to_ascii=False,
|
|
# lower=True,
|
|
# no_line_breaks=True,
|
|
# no_urls=True,
|
|
# no_emails=True,
|
|
# no_phone_numbers=True,
|
|
# no_numbers=False,
|
|
# no_digits=False,
|
|
# no_currency_symbols=True,
|
|
# no_punct=False,
|
|
# replace_with_url="",
|
|
# replace_with_email="",
|
|
# replace_with_phone_number="",
|
|
# replace_with_number="",
|
|
# replace_with_digit="0",
|
|
# replace_with_currency_symbol="",
|
|
# )
|
|
text = clean(text,
|
|
extra_spaces = True,
|
|
lowercase = True
|
|
)
|
|
|
|
# cleaning htmls
|
|
text = cleanhtml(text)
|
|
|
|
# normalizing
|
|
#text = normalizer.normalize(text)
|
|
|
|
# removing wierd patterns
|
|
text = wierd_pattern.sub(r'', text)
|
|
|
|
# removing extra spaces, hashtags
|
|
text = re.sub("#", "", text)
|
|
text = re.sub("\s+", " ", text)
|
|
|
|
return text
|