diff --git a/tokenizers/src/pre_tokenizers/mod.rs b/tokenizers/src/pre_tokenizers/mod.rs index b5d7672e7..fc8b9a0cf 100644 --- a/tokenizers/src/pre_tokenizers/mod.rs +++ b/tokenizers/src/pre_tokenizers/mod.rs @@ -110,10 +110,13 @@ mod tests { ) .unwrap(); - let mut expected_pre_tokenizer = Metaspace::new('▁', true); assert_eq!( pre_tokenizer, - PreTokenizerWrapper::Metaspace(expected_pre_tokenizer) + PreTokenizerWrapper::Metaspace(Metaspace::new_with_prepend_scheme( + '▁', + true, + metaspace::PrependScheme::First + )) ); let pre_tokenizer: PreTokenizerWrapper = serde_json::from_str(