update

tokenizers/src/processors/mod.rs

@@ -17,7 +17,6 @@ use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, PartialEq, Debug, Clone, Eq, Display)]
 #[serde(untagged)]
-#[display(fmt = "{}")]
 pub enum PostProcessorWrapper {
     // Roberta must be before Bert for deserialization (serde does not validate tags)
     Roberta(RobertaProcessing),

tokenizers/src/processors/sequence.rs

@@ -6,8 +6,14 @@ use serde::{Deserialize, Serialize};
 #[macro_rules_attribute(impl_serde_type!)]
 #[derive(Clone, Debug, PartialEq, Eq, Display)]
 #[display(
-    fmt = "[{}]",
-    "processors.iter().map(|d| d.to_string()).collect::<Vec<_>>().join(\", \")"
+    fmt = "processors.Sequence([{}])",
+    "processors.iter().fold(String::new(), |mut acc, p| {
+    if !acc.is_empty() {
+        acc.push_str(\", \");
+    }
+    acc.push_str(&p.to_string());
+    acc
+})"
 )]
 pub struct Sequence {
     processors: Vec<PostProcessorWrapper>,

tokenizers/src/tokenizer/added_vocabulary.rs

@@ -141,7 +141,7 @@ fn space_rightmost_at_start(sentence: &str) -> usize {
 /// exist as required.
 ///
 #[derive(Clone, Debug, Display)]
-#[display(fmt="AddedVocabulary(added_tokens_map_r={{{}}}, encode_special_tokens={})", "&(0..=5).fold(String::new(), |mut acc, key| {if let Some(token) = added_tokens_map_r.get(&key){if !acc.is_empty(){acc.push_str(\", \");}acc.push_str(&format!(\"\n\t{}: {}\", key, &token.to_string()));}acc})", encode_special_tokens)]
+#[display(fmt="AddedVocabulary(added_tokens_map_r={{{}, ...}}, encode_special_tokens={})", "&(0..=5).fold(String::new(), |mut acc, key| {if let Some(token) = added_tokens_map_r.get(&key){if !acc.is_empty(){acc.push_str(\", \");}acc.push_str(&format!(\"\n\t{}: {}\", key, &token.to_string()));}acc})", encode_special_tokens)]
 pub struct AddedVocabulary {
     /// Contains the mapping from String (token content) to ID. This map contains both special
     /// tokens and classic added tokens that were added to the this vocabulary.