From 8d4fae818032b3dd17ce5ecca22a4e185d30aeb1 Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Fri, 12 Jul 2024 10:12:29 +0200 Subject: [PATCH] pass compilation --- tokenizers/src/tokenizer/added_vocabulary.rs | 6 +++--- tokenizers/src/tokenizer/mod.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tokenizers/src/tokenizer/added_vocabulary.rs b/tokenizers/src/tokenizer/added_vocabulary.rs index b5195989d..e5f9adeeb 100644 --- a/tokenizers/src/tokenizer/added_vocabulary.rs +++ b/tokenizers/src/tokenizer/added_vocabulary.rs @@ -5,7 +5,7 @@ use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind}; use regex::Regex; use serde::{ser::SerializeSeq, Deserialize, Serialize, Serializer}; use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, sync::{Arc, Mutex}, }; /// Represent a token added by the user on top of the existing Model vocabulary. @@ -301,9 +301,9 @@ impl AddedVocabulary { } /// Get the token matching the given id if it exists - pub fn simple_id_to_token(&self, id: &u32) -> Option { + pub fn simple_id_to_token(&self, id: u32) -> Option { let added_tokens_map_r = self.added_tokens_map_r.lock().unwrap(); - let token = added_tokens_map_r.get(id).map(|t| t.content.clone()); + let token = added_tokens_map_r.get(&id).map(|t| t.content.clone()); token } diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs index c19c2d5d2..5a1e88110 100644 --- a/tokenizers/src/tokenizer/mod.rs +++ b/tokenizers/src/tokenizer/mod.rs @@ -668,7 +668,7 @@ where if !added_vocab.is_empty() { final_vocab.reserve(added_vocab.len()); for (token, id) in added_vocab { - final_vocab.insert(token.clone(), *id); + final_vocab.insert(token.clone(), id); } } }