90 lines
2.2 KiB
Python
90 lines
2.2 KiB
Python
import hazm
|
||
from cleantext import clean
|
||
import re
|
||
|
||
def cleanhtml(raw_html):
|
||
cleanr = re.compile('<.*?>')
|
||
cleantext = re.sub(cleanr, '', raw_html)
|
||
return cleantext
|
||
|
||
normalizer = hazm.Normalizer()
|
||
wierd_pattern = re.compile("["
|
||
u"\U0001F600-\U0001F64F" # emoticons
|
||
u"\U0001F300-\U0001F5FF" # symbols & pictographs
|
||
u"\U0001F680-\U0001F6FF" # transport & map symbols
|
||
u"\U0001F1E0-\U0001F1FF" # flags (iOS)
|
||
u"\U00002702-\U000027B0"
|
||
u"\U000024C2-\U0001F251"
|
||
u"\U0001f926-\U0001f937"
|
||
u'\U00010000-\U0010ffff'
|
||
u"\u200d"
|
||
u"\u2640-\u2642"
|
||
u"\u2600-\u2B55"
|
||
u"\u23cf"
|
||
u"\u23e9"
|
||
u"\u231a"
|
||
u"\u3030"
|
||
u"\ufe0f"
|
||
u"\u2069"
|
||
u"\u2066"
|
||
# u"\u200c" # نیم فاصله ها حذف نشود
|
||
u"\u2068"
|
||
u"\u2067"
|
||
"]+", flags=re.UNICODE)
|
||
|
||
def cleaning(text):
|
||
text = text.strip()
|
||
|
||
# regular cleaning
|
||
# text = clean(text,
|
||
# fix_unicode=True,
|
||
# to_ascii=False,
|
||
# lower=True,
|
||
# no_line_breaks=True,
|
||
# no_urls=True,
|
||
# no_emails=True,
|
||
# no_phone_numbers=True,
|
||
# no_numbers=False,
|
||
# no_digits=False,
|
||
# no_currency_symbols=True,
|
||
# no_punct=False,
|
||
# replace_with_url="",
|
||
# replace_with_email="",
|
||
# replace_with_phone_number="",
|
||
# replace_with_number="",
|
||
# replace_with_digit="0",
|
||
# replace_with_currency_symbol="",
|
||
# )
|
||
text = clean(text,
|
||
extra_spaces = True,
|
||
lowercase = True
|
||
)
|
||
|
||
# cleaning htmls
|
||
text = cleanhtml(text)
|
||
|
||
# normalizing
|
||
text = normalizer.normalize(text)
|
||
|
||
# removing wierd patterns
|
||
text = wierd_pattern.sub(r'', text)
|
||
|
||
# removing extra spaces, hashtags
|
||
text = re.sub("#", "", text)
|
||
text = re.sub("\s+", " ", text)
|
||
|
||
return text
|
||
|
||
|
||
|
||
|
||
|
||
# with open('./ghavanins.txt', encoding="utf-8") as fp:
|
||
# current_content = fp.read()
|
||
|
||
# current_content = cleaning(current_content)
|
||
|
||
|
||
# with open('./ghavanins2.txt', 'wb') as f:
|
||
# f.write(current_content.encode('utf-8', 'ignore'))
|