From 7b9537dd55f4cf299d9f15fe01c8cf6845146f4d Mon Sep 17 00:00:00 2001 From: m_ghazizadeh Date: Mon, 30 Jun 2025 18:24:43 +0330 Subject: [PATCH] Remove additional [] in two regex codes where doens't needed! --- tokenizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokenizer.py b/tokenizer.py index 59694cd..946a7ad 100644 --- a/tokenizer.py +++ b/tokenizer.py @@ -16,7 +16,7 @@ class Tokenizer(): nums_list = re.findall(pattern, doc_string) doc_string = re.sub(pattern, 'floatingpointnumber', doc_string) - pattern = r'([!\.\?؟]+)[\n]*' + pattern = r'([!\.\?؟]+)\n*' tmp = re.findall(pattern, doc_string) doc_string = re.sub(pattern, self.add_tab, doc_string) @@ -32,7 +32,7 @@ class Tokenizer(): tmp = re.findall(pattern, doc_string) doc_string = re.sub(pattern, self.add_tab, doc_string) - pattern = r'[\n]+' + pattern = r'\n+' doc_string = re.sub(pattern, self.add_tab, doc_string) for number in nums_list: