fmt.

huggingface · Aug 8, 2024 · 3c6ee87 · 3c6ee87
1 parent b617b18
commit 3c6ee87
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tokenizers/src/tokenizer/pre_tokenizer.rs b/tokenizers/src/tokenizer/pre_tokenizer.rs
@@ -154,7 +154,7 @@ impl PreTokenizedString {
                         .flat_map(|split| {
                             split.tokens.unwrap().into_iter().map(|token| {
                                 // Replace this with the actual fields you need for the Encoding type
-                                (token.id,String::with_capacity(0), (0, 0), None, 0)
+                                (token.id, String::with_capacity(0), (0, 0), None, 0)
                             })
                         })
                         .collect();