Merge branch 'add-display' of github.com:huggingface/tokenizers into …

…add-display

fix git suggestion

nit

__repr__ should use Debug?

small updates

Simple lazygit test

bindings/python/src/pre_tokenizers.rs

@@ -664,7 +664,15 @@ impl Serialize for PyPreTokenizerWrapper {
 #[serde(untagged)]
 // #[display(fmt = "")]
 pub(crate) enum PyPreTokenizerTypeWrapper {
-    #[display(fmt = "[{}]", "_0.iter().map(|d| d.as_ref().read().unwrap().to_string()).collect::<Vec<_>>().join(\", \")")]
+    #[display(fmt = "[{}]", "_0_0.iter()
+    .map(|d| d.as_ref().read().unwrap().to_string())
+    .fold(String::new(), |mut acc, s| {
+        if !acc.is_empty() {
+            acc.push_str(", ");
+        }
+        acc.push_str(&s);
+        acc
+    })")]
     Sequence(Vec<Arc<RwLock<PyPreTokenizerWrapper>>>),
     #[display(fmt ="{}", "_0.as_ref().read().unwrap()")]
     Single(Arc<RwLock<PyPreTokenizerWrapper>>),

tokenizers/src/lib.rs

@@ -23,7 +23,7 @@
 //! 4. The `PostProcessor`: in charge of post-processing the `Encoding` to add anything relevant
 //!    that, for example, a language model would need, such as special tokens.
 //!
-//! ## Loading a pretrained tokenizer from the Hub
+//! ## Loading a pretrained tokenizer from the Hub.
 //! ```
 //! use tokenizers::tokenizer::{Result, Tokenizer};
 //!
@@ -39,7 +39,7 @@
 //! }
 //! ```
 //!
-//! ## Deserialization and tokenization example
+//! ## Deserialization and tokenization example.
 //!
 //! ```no_run
 //! use tokenizers::tokenizer::{Result, Tokenizer, EncodeInput};