Update tokenizers/src/tokenizer/added_vocabulary.rs

Co-authored-by: Luc Georges <[email protected]>
huggingface · Jul 12, 2024 · 0594b51 · 0594b51
1 parent d6123a6
commit 0594b51
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/tokenizers/src/tokenizer/added_vocabulary.rs b/tokenizers/src/tokenizer/added_vocabulary.rs
@@ -223,7 +223,6 @@ impl AddedVocabulary {
             .or_else(|| model.id_to_token(id))
     }
 
-    //
     pub fn simple_id_to_token(&self, id: u32) -> Option<String> {
         self.added_tokens_map_r.get(&id).map(|t| t.content.clone())
     }