Merge branch 'add-display' of github.com:huggingface/tokenizers into …

…add-display fix git suggestion nit __repr__ should use Debug? small updates
huggingface · Jun 6, 2024 · 0d9a452 · 0d9a452
2 parents 27576e5 + e48cd3a
commit 0d9a452
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs
@@ -664,7 +664,15 @@ impl Serialize for PyPreTokenizerWrapper {
 #[serde(untagged)]
 // #[display(fmt = "")]
 pub(crate) enum PyPreTokenizerTypeWrapper {
-    #[display(fmt = "[{}]", "_0.iter().map(|d| d.as_ref().read().unwrap().to_string()).collect::<Vec<_>>().join(\", \")")]
+    #[display(fmt = "[{}]", "_0_0.iter()
+    .map(|d| d.as_ref().read().unwrap().to_string())
+    .fold(String::new(), |mut acc, s| {
+        if !acc.is_empty() {
+            acc.push_str(", ");
+        }
+        acc.push_str(&s);
+        acc
+    })")]
     Sequence(Vec<Arc<RwLock<PyPreTokenizerWrapper>>>),
     #[display(fmt ="{}", "_0.as_ref().read().unwrap()")]
     Single(Arc<RwLock<PyPreTokenizerWrapper>>),