nits

huggingface · Nov 13, 2023 · 34355e5 · 34355e5
1 parent 3ec7b54
commit 34355e5
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/tokenizers/src/pre_tokenizers/mod.rs b/tokenizers/src/pre_tokenizers/mod.rs
@@ -110,10 +110,13 @@ mod tests {
         )
         .unwrap();
 
-        let mut expected_pre_tokenizer = Metaspace::new('▁', true);
         assert_eq!(
             pre_tokenizer,
-            PreTokenizerWrapper::Metaspace(expected_pre_tokenizer)
+            PreTokenizerWrapper::Metaspace(Metaspace::new_with_prepend_scheme(
+                '▁',
+                true,
+                metaspace::PrependScheme::First
+            ))
         );
 
         let pre_tokenizer: PreTokenizerWrapper = serde_json::from_str(