fmt

huggingface · Aug 19, 2024 · fcb7af0 · fcb7af0
1 parent cbbbd20
commit fcb7af0
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs
@@ -1188,7 +1188,9 @@ where
                         s
                     }),
                     |seq| {
-                        let normalized = self.added_vocabulary.extract_and_normalize(self.normalizer.as_ref(), seq.as_ref());
+                        let normalized = self
+                            .added_vocabulary
+                            .extract_and_normalize(self.normalizer.as_ref(), seq.as_ref());
                         let pre_tokenized = self.do_pre_tokenize(normalized)?;
                         Ok(pre_tokenized
                             .get_splits(OffsetReferential::Original, OffsetType::Byte)
@@ -1240,7 +1242,9 @@ where
                 s
             }),
             |seq| {
-                let normalized = self.added_vocabulary.extract_and_normalize(self.normalizer.as_ref(), seq.as_ref());
+                let normalized = self
+                    .added_vocabulary
+                    .extract_and_normalize(self.normalizer.as_ref(), seq.as_ref());
                 let pre_tokenized = self.do_pre_tokenize(normalized)?;
                 Ok(pre_tokenized
                     .get_splits(OffsetReferential::Original, OffsetType::Byte)