remove prints

huggingface · Jul 11, 2024 · 1639ae2 · 1639ae2
1 parent 7e83218
commit 1639ae2
Show file tree

Hide file tree

Showing 3 changed files with 0 additions and 3 deletions.
diff --git a/tokenizers/src/tokenizer/added_vocabulary.rs b/tokenizers/src/tokenizer/added_vocabulary.rs
@@ -487,7 +487,6 @@ impl AddedVocabulary {
                 Ok(self.split_with_indices(sequence, &self.split_normalized_trie))
             })
             .expect("AddedVocabulary bad split");
-        println!("after pretokenized.split {:?}", pretokenized);
         // ["I read a book", "   <s>", "Hey"] -> ["▁I read a book", "▁   <s>", "▁Hey"]
         // ["▁I read a book", "▁   <s>", "▁Hey"] -> [.., "▁   ", "<s>", "▁Hey"]
 

diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs
@@ -943,7 +943,6 @@ where
         pretokenized: P,
     ) -> Result<PreTokenizedString> {
         let mut pretokenized: PreTokenizedString = pretokenized.into();
-        println!("do pretokenized converted: {:?}", pretokenized);
         if let Some(ref pretok) = self.pre_tokenizer {
             pretok.pre_tokenize(&mut pretokenized)?;
         }

diff --git a/tokenizers/src/tokenizer/pre_tokenizer.rs b/tokenizers/src/tokenizer/pre_tokenizer.rs
@@ -166,7 +166,6 @@ impl PreTokenizedString {
                         if let Some(converter) = offset_converter {
                             offsets = converter.convert(offsets).unwrap_or(offsets);
                         }
-                        println!("into encode, {:?}, {:?}", normalized.clone(), token.value);
                         (
                             token.id,
                             token.value,