update sequence for pre_tokenizers using fold

huggingface · Jun 10, 2024 · 7591f2b · 7591f2b
1 parent 93a1e63
commit 7591f2b
Showing 1 changed file with 8 additions and 2 deletions.
diff --git a/tokenizers/src/pre_tokenizers/sequence.rs b/tokenizers/src/pre_tokenizers/sequence.rs
@@ -7,8 +7,14 @@ use serde::{Deserialize, Serialize};
 #[macro_rules_attribute(impl_serde_type!)]
 #[derive(Clone, Debug, PartialEq, Display)]
 #[display(
-    fmt = "[{}]",
-    "pretokenizers.iter().map(|d| d.to_string()).collect::<Vec<_>>().join(\", \")"
+    fmt = "pre_tokenizers.Seqence([{}])",
+    "pretokenizers.iter().fold(String::new(), |mut acc, p| {
+    if !acc.is_empty(){
+       acc.push_str(\", \")
+    }
+    acc.push_str(&p.to_string());
+    acc
+    })"
 )]
 pub struct Sequence {
     pretokenizers: Vec<PreTokenizerWrapper>,