update

huggingface · Jun 10, 2024 · 4a34870 · 4a34870
1 parent 9c45e8f
commit 4a34870
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/tokenizers/src/decoders/mod.rs b/tokenizers/src/decoders/mod.rs
@@ -25,7 +25,7 @@ use derive_more::Display;
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Clone, Debug, Display)]
-#[display(fmt="decoders.{})]
+#[display(fmt="decoders.{}")]
 #[serde(untagged)]
 pub enum DecoderWrapper {
     BPE(BPEDecoder),

diff --git a/tokenizers/src/models/wordpiece/mod.rs b/tokenizers/src/models/wordpiece/mod.rs
@@ -121,8 +121,14 @@ impl WordPieceBuilder {
 /// model.
 #[derive(Clone, PartialEq, Eq, Display)]
 #[display(
-    fmt = "vocab={:?}, unk_token={}, continuing_subword_prefix={:?}",
-    vocab,
+    fmt = "vocab={}, unk_token={}, continuing_subword_prefix={:?}",
+    "vocab.iter().take(5).fold(String::new(), |mut acc, (key, value)| {
+        if !acc.is_empty() {
+            acc.push_str(\", \");
+        }
+        acc.push_str(&format!(\"\'{}\': {}\", key, value));
+        acc
+    })",
     unk_token,
     continuing_subword_prefix
 )]