fix build

huggingface · Jul 19, 2024 · 2048c02 · 2048c02
1 parent c2a320c
commit 2048c02
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 9 deletions.
diff --git a/tokenizers/src/normalizers/mod.rs b/tokenizers/src/normalizers/mod.rs
@@ -18,7 +18,6 @@ use serde::{Deserialize, Serialize};
 
 use crate::{NormalizedString, Normalizer};
 use pyo3_special_method_derive::AutoDisplay;
-use serde::{Deserialize, Serialize};
 
 /// Wrapper for known Normalizers.
 #[derive(Clone, Debug, Deserialize, Serialize, AutoDisplay)]

diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs
@@ -1347,13 +1347,6 @@ where
 #[cfg(test)]
 mod tests {
     use super::Tokenizer;
-
-    #[cfg(feature = "http")]
-    #[test]
-    fn test_from_pretrained() {
-        let tok = Tokenizer::from_pretrained("Qwen/Qwen2-7B-Instruct".to_string(), None);
-        println!("ROCK!")
-
     use crate::AddedToken;
     use crate::Tokenizer;
 
@@ -1407,4 +1400,4 @@ mod tests {
         let decoded = tokenizer.decode(encoded.get_ids(), false);
         assert_eq!(decoded.unwrap(), "Hey! how is this token: д")
     }
-}
+}