[pre_tokenizers] Fix sentencepiece based Metaspace

bindings/python/src/pre_tokenizers.rs

@@ -489,6 +489,16 @@ impl PyMetaspace {
         setter!(self_, Metaspace, add_prefix_space, add_prefix_space);
     }
 
+    #[getter]
+    fn get_use_legacy(self_: PyRef<Self>) -> bool {
+        getter!(self_, Metaspace, legacy)
+    }
+
+    #[setter]
+    fn set_use_legacy(self_: PyRef<Self>, legacy: bool) {
+        setter!(self_, Metaspace, legacy, legacy);
+    }
+
     #[new]
     #[pyo3(signature = (replacement = PyChar('▁'), add_prefix_space = true, **_kwargs), text_signature = "(self, replacement=\"_\", add_prefix_space=True)")]
     fn new(

tokenizers/src/pre_tokenizers/metaspace.rs

@@ -9,6 +9,7 @@ use crate::tokenizer::{Decoder, PreTokenizedString, PreTokenizer, Result, SplitD
 pub struct Metaspace {
     replacement: char,
     pub add_prefix_space: bool,
+    pub legacy:bool,
     #[serde(skip)]
     str_rep: String,
 }
@@ -29,6 +30,8 @@ impl<'de> Deserialize<'de> for Metaspace {
             _type: Type,
             replacement: char,
             pub add_prefix_space: bool,
+            // TODO Make sure legacy does not have to be here for forward compatibility!
+            pub legacy: bool,
             #[serde(skip, rename = "str_rep")]
             _str_rep: String,
         }
@@ -44,6 +47,7 @@ impl Metaspace {
             replacement,
             str_rep: replacement.to_string(),
             add_prefix_space,
+            legacy: true,
         }
     }
 
@@ -65,12 +69,17 @@ impl Default for Metaspace {
 
 impl PreTokenizer for Metaspace {
     fn pre_tokenize(&self, pretokenized: &mut PreTokenizedString) -> Result<()> {
+        let mut first_split = true;
+
         pretokenized.split(|_, mut normalized| {
             normalized.replace(' ', &self.str_rep)?;
-            if self.add_prefix_space && !normalized.get().starts_with(self.replacement) {
-                normalized.prepend(&self.str_rep);
+            if self.add_prefix_space{
+                if  self.legacy{normalized.prepend(&self.str_rep);}
+                else if self.add_prefix_space && first_split{
+                    normalized.prepend(&self.str_rep);
+                    first_split = false; // Set the flag to false after the first split
+                }
             }
-
             normalized.split(self.replacement, SplitDelimiterBehavior::MergedWithNext)
         })
     }

tokenizers/src/tokenizer/mod.rs

@@ -658,6 +658,10 @@ where
 
         final_vocab
     }
+    /// Get the added tokens encoder
+    pub fn get_added_tokens_encoder(&self) -> HashMap<String, u32> {
+        self.added_vocabulary.get_vocab().clone()
+    }
 
     /// Get the added tokens decoder
     pub fn get_added_tokens_decoder(&self) -> HashMap<u32, AddedToken> {
@@ -693,6 +697,7 @@ where
         offsets_type: OffsetType,
     ) -> Result<Encoding> {
         let encode = |is_pre_tokenized, subseq_idx, subseq| -> Result<Encoding> {
+            // FIXME FIXME FIXME all of our SPM issues most probably come from here. The addition of the space after special tokens
             let normalized = self
                 .added_vocabulary
                 .extract_and_normalize(self.normalizer.as_ref(), subseq);