From 8f4b4680cb47906c33f1d8f27b8e722c9deb2163 Mon Sep 17 00:00:00 2001
From: = <saedi.behnam.336@gmail.com>
Date: Wed, 11 Aug 2021 13:02:21 +0430
Subject: [PATCH 1/2] Fix some tokenization issues

---
 hazm/WordTokenizer.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/hazm/WordTokenizer.py b/hazm/WordTokenizer.py
index cda1e986..89a90ed6 100644
--- a/hazm/WordTokenizer.py
+++ b/hazm/WordTokenizer.py
@@ -52,16 +52,19 @@ def __init__(self, words_file=default_words, verbs_file=default_verbs, join_verb
 										u"\U0001F4CC\U0001F4CD"  # other emojis
 										"]", flags=re.UNICODE)
 		self.emoji_repl = r'\g<0> '
-		self.id_pattern = re.compile(r'([^\w\._]+)(@[\w_]+)')
-		self.id_repl = r'\1ID'
-		self.link_pattern = re.compile(r'((https?|ftp):\/\/)?(?<!@)([wW]{3}\.)?(([\w-]+)(\.(\w){2,})+([-\w@:%_\+\/~#?&]+)?)')
-		self.link_repl = r'LINK'
+		self.id_pattern = re.compile(r'(?<![\w\._])(@[\w_]+)')
+		self.id_repl = r' ID '
+		self.link_pattern = re.compile(r'((https?|ftp):\/\/)?(?<!@)(([\w-]+\.)+((?![\d۰-۹])\w)+)[-\w@:%_\.\+\/~#?=&]*')
+		self.link_repl = r' LINK '
 		self.email_pattern = re.compile(r'[a-zA-Z0-9\._\+-]+@([a-zA-Z0-9-]+\.)+[A-Za-z]{2,}')
-		self.email_repl = r'EMAIL'
-		self.number_int_pattern = re.compile(r'([^\.,\w]+)([\d۰-۹]+)([^\.,\w]+)')
-		self.number_int_repl = lambda m: m.group(1) + 'NUM'+ str(len(m.group(2))) + m.group(3)
-		self.number_float_pattern = re.compile(r'([^,\w]+)([\d۰-۹,]+[\.٫]{1}[\d۰-۹]+)([^,\w]+)')
-		self.number_float_repl = r'\1NUMF\3'
+		self.email_repl = r' EMAIL '
+
+		# '٫' is the decimal separator and '٬' is the thousands separator
+		self.number_int_pattern = re.compile(r'\b(?<![\d۰-۹][\.٫٬,])([\d۰-۹]+)(?![\.٫٬,][\d۰-۹])\b')
+		self.number_int_repl = lambda m: ' NUM' + str(len(m.group(1))) + ' '
+		self.number_float_pattern = re.compile(r'\b(?<!\.)([\d۰-۹,٬]+[\.٫٬]{1}[\d۰-۹]+)\b(?!\.)')
+		self.number_float_repl = r' NUMF '
+
 		self.hashtag_pattern = re.compile(r'\#([\S]+)')
 		# NOTE: python2.7 does not support unicodes with \w
 
@@ -96,12 +99,12 @@ def tokenize(self, text):
 
 		if self.separate_emoji:
 			text = self.emoji_pattern.sub(self.emoji_repl, text)
+		if self.replace_emails:
+			text = self.email_pattern.sub(self.email_repl, text)
 		if self.replace_links:
 			text = self.link_pattern.sub(self.link_repl, text)
 		if self.replace_IDs:
 			text = self.id_pattern.sub(self.id_repl, text)
-		if self.replace_emails:
-			text = self.email_pattern.sub(self.email_repl, text)
 		if self.replace_hashtags:
 			text = self.hashtag_pattern.sub(self.hashtag_repl, text)
 		if self.replace_numbers:

From 5704408fbb966724ea9660f761a339d9317d1b50 Mon Sep 17 00:00:00 2001
From: = <saedi.behnam.336@gmail.com>
Date: Wed, 11 Aug 2021 20:37:52 +0430
Subject: [PATCH 2/2] Only allow english letters in link token

---
 hazm/WordTokenizer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hazm/WordTokenizer.py b/hazm/WordTokenizer.py
index 89a90ed6..31a88487 100644
--- a/hazm/WordTokenizer.py
+++ b/hazm/WordTokenizer.py
@@ -54,7 +54,7 @@ def __init__(self, words_file=default_words, verbs_file=default_verbs, join_verb
 		self.emoji_repl = r'\g<0> '
 		self.id_pattern = re.compile(r'(?<![\w\._])(@[\w_]+)')
 		self.id_repl = r' ID '
-		self.link_pattern = re.compile(r'((https?|ftp):\/\/)?(?<!@)(([\w-]+\.)+((?![\d۰-۹])\w)+)[-\w@:%_\.\+\/~#?=&]*')
+		self.link_pattern = re.compile(r'((https?|ftp):\/\/)?(?<!@)(([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,})[-\w@:%_\.\+\/~#?=&]*')
 		self.link_repl = r' LINK '
 		self.email_pattern = re.compile(r'[a-zA-Z0-9\._\+-]+@([a-zA-Z0-9-]+\.)+[A-Za-z]{2,}')
 		self.email_repl = r' EMAIL '