nits

huggingface · Dec 18, 2023 · a581a04 · a581a04
1 parent ed302a8
commit a581a04
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/tokenizers/src/tokenizer/added_vocabulary.rs b/tokenizers/src/tokenizer/added_vocabulary.rs
@@ -183,7 +183,7 @@ impl AddedVocabulary {
             special_tokens_set: HashSet::new(),
             split_trie: (trie, vec![]),
             split_normalized_trie: (normalized_trie, vec![]),
-            encode_special_tokens: false,
+            encode_special_tokens: true,
         }
     }
     /// Size of the additional vocabulary
@@ -370,7 +370,7 @@ impl AddedVocabulary {
             let id = split_re.1[aho_id];
             let added_token = &self.added_tokens_map_r.get(&id).unwrap();
 
-            if self.encode_special_tokens && added_token.special {
+            if self.encode_special_tokens && self.special_tokens_set.contains(&added_token.content) {
                 continue;
             }
 

diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs
@@ -686,7 +686,11 @@ where
     pub fn id_to_token(&self, id: u32) -> Option<String> {
         self.added_vocabulary.id_to_token(id, &self.model)
     }
-
+
+    /// set the added bocab's splitting scheme
+    pub fn set_encode_special_tokens(&mut self, value:bool){
+        self.added_vocabulary.set_encode_special_tokens(value);
+    }
     /// Encode a single sequence
     fn encode_single_sequence(
         &self,