ajokar/Flair_NER
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove additional [] in two regex codes where doens't needed!

Browse Source
This commit is contained in:
m_ghazizadeh 2025-06-30 18:24:43 +03:30
parent c3b0f1c61c
commit 7b9537dd55
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tokenizer.py
View File

@ -16,7 +16,7 @@ class Tokenizer():
nums_list = re.findall(pattern, doc_string)
doc_string = re.sub(pattern, 'floatingpointnumber', doc_string)
pattern = r'([!\.\?؟]+)[\n]*'
pattern = r'([!\.\?؟]+)\n*'
tmp = re.findall(pattern, doc_string)
doc_string = re.sub(pattern, self.add_tab, doc_string)
@ -32,7 +32,7 @@ class Tokenizer():
tmp = re.findall(pattern, doc_string)
doc_string = re.sub(pattern, self.add_tab, doc_string)
pattern = r'[\n]+'
pattern = r'\n+'
doc_string = re.sub(pattern, self.add_tab, doc_string)
for number in nums_list: