Small performance fixup (negligible but obviously better).

huggingface · Aug 1, 2024 · 41f258c · 41f258c
1 parent 835f9a1
commit 41f258c
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/tokenizers/src/models/bpe/model.rs b/tokenizers/src/models/bpe/model.rs
@@ -460,14 +460,14 @@ impl BPE {
     }
 
     fn tokenize_with_cache(&self, sequence: &str) -> Result<Vec<Token>> {
-        if let Some(ref hit) = self.cache.as_ref().and_then(|c| c.get(sequence)) {
-            return Ok(self.word_to_tokens(hit).collect());
-        }
         if self.ignore_merges {
             if let Some(id) = self.vocab.get(sequence) {
                 return Ok(vec![Token::new(*id, sequence.to_string().clone(), (0, 0))]);
             }
         }
+        if let Some(ref hit) = self.cache.as_ref().and_then(|c| c.get(sequence)) {
+            return Ok(self.word_to_tokens(hit).collect());
+        }
         let word = self.merge_word(sequence)?;
         let ret = self.word_to_tokens(&word).collect();
         if let Some(ref cache) = self.cache {