From 590648ebca3782da1aacb94f389f67173794eca1 Mon Sep 17 00:00:00 2001
From: TyrianOtter <154254636+TyrianOtter@users.noreply.github.com>
Date: Tue, 1 Oct 2024 14:02:03 -0400
Subject: [PATCH] fix `CLIPTokenizer` skipping underscores

---
 src/refiners/foundationals/clip/tokenizer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/refiners/foundationals/clip/tokenizer.py b/src/refiners/foundationals/clip/tokenizer.py
index 9df0fcfae..1a1507aa7 100644
--- a/src/refiners/foundationals/clip/tokenizer.py
+++ b/src/refiners/foundationals/clip/tokenizer.py
@@ -44,7 +44,7 @@ def __init__(
         # to get rid of the dependence on the `regex` module. Unicode support could
         # potentially be added back by leveraging the `\w` character class.
         self.token_pattern = re.compile(
-            pattern=r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[a-zA-Z]+|[0-9]|[^\s\w]+""",
+            pattern=r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[a-zA-Z]+|[0-9]|(?:[^\s\w]|_)+""",
             flags=re.IGNORECASE,
         )
         self.start_of_text_token_id: int = start_of_text_token_id