From e48cd3aadedeb94bc54ded2c48487f876e781fe1 Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Thu, 6 Jun 2024 09:33:31 +0200 Subject: [PATCH] Update bindings/python/src/pre_tokenizers.rs --- bindings/python/src/pre_tokenizers.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs index cb4f92ff1..113a90d9c 100644 --- a/bindings/python/src/pre_tokenizers.rs +++ b/bindings/python/src/pre_tokenizers.rs @@ -664,7 +664,15 @@ impl Serialize for PyPreTokenizerWrapper { #[serde(untagged)] // #[display(fmt = "")] pub(crate) enum PyPreTokenizerTypeWrapper { - #[display(fmt = "[{}]", "_0.iter().map(|d| d.as_ref().read().unwrap().to_string()).collect::>().join(\", \")")] + #[display(fmt = "[{}]", "_0_0.iter() + .map(|d| d.as_ref().read().unwrap().to_string()) + .fold(String::new(), |mut acc, s| { + if !acc.is_empty() { + acc.push_str(", "); + } + acc.push_str(&s); + acc + })")] Sequence(Vec>>), #[display(fmt ="{}", "_0.as_ref().read().unwrap()")] Single(Arc>),