styling

huggingface · Jun 10, 2024 · 35d442d · 35d442d
1 parent 0a16ca0
commit 35d442d
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 4 deletions.
diff --git a/tokenizers/src/models/wordlevel/mod.rs b/tokenizers/src/models/wordlevel/mod.rs
@@ -95,13 +95,17 @@ impl WordLevelBuilder {
 }
 
 #[derive(PartialEq, Clone, Eq, Display)]
-#[display(fmt = "vocab={{{}, ...}}, unk_token={}", "vocab.iter().take(5).fold(String::new(), |mut acc, (key, value)| {
+#[display(
+    fmt = "vocab={{{}, ...}}, unk_token={}",
+    "vocab.iter().take(5).fold(String::new(), |mut acc, (key, value)| {
     if !acc.is_empty() {
         acc.push_str(\", \");
     }
     acc.push_str(&format!(\"\'{}\': {}\", key, value));
     acc
-})", unk_token)]
+})",
+    unk_token
+)]
 pub struct WordLevel {
     vocab: HashMap<String, u32>,
     vocab_r: HashMap<u32, String>,

diff --git a/tokenizers/src/utils/padding.rs b/tokenizers/src/utils/padding.rs
@@ -20,7 +20,15 @@ impl std::convert::AsRef<str> for PaddingDirection {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Display)]
-#[display(fmt="strategy={}, direction={}, pad_to_multiple_of={}, pad_id={}, pad_type_id={}, pad_token={}", strategy, direction, "pad_to_multiple_of.unwrap()", pad_id, pad_type_id, pad_token)]
+#[display(
+    fmt = "strategy={}, direction={}, pad_to_multiple_of={}, pad_id={}, pad_type_id={}, pad_token={}",
+    strategy,
+    direction,
+    "pad_to_multiple_of.unwrap()",
+    pad_id,
+    pad_type_id,
+    pad_token
+)]
 pub struct PaddingParams {
     pub strategy: PaddingStrategy,
     pub direction: PaddingDirection,

diff --git a/tokenizers/src/utils/truncation.rs b/tokenizers/src/utils/truncation.rs
@@ -1,6 +1,6 @@
 use crate::tokenizer::{Encoding, Result};
-use display_derive::StructDisplay;
 use derive_more::Display;
+use display_derive::StructDisplay;
 use serde::{Deserialize, Serialize};
 use std::cmp;
 use std::mem;