From 0c94da5df19f93d1891eb5c9b6bfde34157ae9f9 Mon Sep 17 00:00:00 2001 From: chris-ha458 Date: Fri, 27 Oct 2023 11:46:17 +0900 Subject: [PATCH] rename modeled `token_to_id` rename modeled `token_to_id` into `token_to_id_model` --- tokenizers/src/tokenizer/added_vocabulary.rs | 6 +++--- tokenizers/src/tokenizer/mod.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tokenizers/src/tokenizer/added_vocabulary.rs b/tokenizers/src/tokenizer/added_vocabulary.rs index 6870b7ec7..575e94653 100644 --- a/tokenizers/src/tokenizer/added_vocabulary.rs +++ b/tokenizers/src/tokenizer/added_vocabulary.rs @@ -199,7 +199,7 @@ impl AddedVocabulary { } /// Get the id matching one of our token if it exists - pub fn token_to_id(&self, token: &str, model: &impl Model) -> Option { + pub fn token_to_id_model(&self, token: &str, model: &impl Model) -> Option { self.added_tokens_map .get(token) .copied() @@ -256,7 +256,7 @@ impl AddedVocabulary { continue; } // If a token is already part of the vocabulary, we mark it as added - let new_id = if let Some(new_id) = self.token_to_id(&token.content, model) { + let new_id = if let Some(new_id) = self.token_to_id_model(&token.content, model) { new_id } else { self.added_tokens_map.values().cloned().max().map_or( @@ -307,7 +307,7 @@ impl AddedVocabulary { .map(|token| { ( token, - self.token_to_id(&token.content, model) + self.token_to_id_model(&token.content, model) .expect("Missing additional token"), ) }) diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs index 2d7e10f73..a5633ceac 100644 --- a/tokenizers/src/tokenizer/mod.rs +++ b/tokenizers/src/tokenizer/mod.rs @@ -677,7 +677,7 @@ where /// Converts a token in the corresponding id. pub fn token_to_id(&self, token: &str) -> Option { - self.added_vocabulary.token_to_id(token, &self.model) + self.added_vocabulary.token_to_id_model(token, &self.model) } /// Converts an id to the corresponding token.