Skip to content

Commit

Permalink
remove prints
Browse files Browse the repository at this point in the history
  • Loading branch information
ArthurZucker committed Aug 5, 2024
1 parent f852e9e commit 024ccc2
Showing 1 changed file with 0 additions and 7 deletions.
7 changes: 0 additions & 7 deletions tokenizers/src/models/bpe/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -282,21 +282,14 @@ pub(crate) fn convert_merges_to_hashmap<I: Iterator<Item = String>>(
_vocab: &Vocab,
) -> Result<Merges> {
let mut merges = vec![];

println!(" Parsing merges!");
let lines = iter.filter(|l| !l.starts_with("#version"));
for (rank, line) in lines.enumerate() {
let parts = line.split(' ').collect::<Vec<_>>();
if parts.len() != 2 {
println!("bad merges parts = {:?}", parts);

return Err(Error::BadMerges(rank + 1).into());
}

println!("parts = {:?}", parts);
merges.push((parts[0].to_string(), parts[1].to_string()));
}
println!(" Finished parsing");
Ok(merges)
}

Expand Down

0 comments on commit 024ccc2

Please sign in to comment.