diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs index cb4f92ff1..113a90d9c 100644 --- a/bindings/python/src/pre_tokenizers.rs +++ b/bindings/python/src/pre_tokenizers.rs @@ -664,7 +664,15 @@ impl Serialize for PyPreTokenizerWrapper { #[serde(untagged)] // #[display(fmt = "")] pub(crate) enum PyPreTokenizerTypeWrapper { - #[display(fmt = "[{}]", "_0.iter().map(|d| d.as_ref().read().unwrap().to_string()).collect::>().join(\", \")")] + #[display(fmt = "[{}]", "_0_0.iter() + .map(|d| d.as_ref().read().unwrap().to_string()) + .fold(String::new(), |mut acc, s| { + if !acc.is_empty() { + acc.push_str(", "); + } + acc.push_str(&s); + acc + })")] Sequence(Vec>>), #[display(fmt ="{}", "_0.as_ref().read().unwrap()")] Single(Arc>), diff --git a/tokenizers/src/lib.rs b/tokenizers/src/lib.rs index eb89b9315..830a1c8a4 100644 --- a/tokenizers/src/lib.rs +++ b/tokenizers/src/lib.rs @@ -23,7 +23,7 @@ //! 4. The `PostProcessor`: in charge of post-processing the `Encoding` to add anything relevant //! that, for example, a language model would need, such as special tokens. //! -//! ## Loading a pretrained tokenizer from the Hub +//! ## Loading a pretrained tokenizer from the Hub. //! ``` //! use tokenizers::tokenizer::{Result, Tokenizer}; //! @@ -39,7 +39,7 @@ //! } //! ``` //! -//! ## Deserialization and tokenization example +//! ## Deserialization and tokenization example. //! //! ```no_run //! use tokenizers::tokenizer::{Result, Tokenizer, EncodeInput};