nit

huggingface · Jul 11, 2024 · 52a6f2e · 52a6f2e
1 parent 6c713d2
commit 52a6f2e
Showing 1 changed file with 0 additions and 10 deletions.
diff --git a/tokenizers/src/pre_tokenizers/metaspace.rs b/tokenizers/src/pre_tokenizers/metaspace.rs
@@ -130,22 +130,12 @@ impl PreTokenizer for Metaspace {
             }
             pretokenized.split(|_, mut normalized| {
                 normalized.replace(' ', &self.str_rep)?;
-                match self.prepend_scheme {
-                    PrependScheme::Always => {
-                        if !normalized.get().starts_with(self.replacement) {
-                            normalized.prepend(&self.str_rep);
-                        }
-                    }
-                    PrependScheme::Never => {}
-                    _ => {} // Handle other cases if needed
-                };
                 if self.split {
                     normalized.split(self.replacement, SplitDelimiterBehavior::MergedWithNext)
                 } else {
                     Ok(vec![normalized])
                 }
             });
-
             Ok(())
         } else {
             pretokenized.split(|_, mut normalized| {