diff --git a/tokenizers/src/tokenizer/pre_tokenizer.rs b/tokenizers/src/tokenizer/pre_tokenizer.rs index 125689a81..0d54cd62b 100644 --- a/tokenizers/src/tokenizer/pre_tokenizer.rs +++ b/tokenizers/src/tokenizer/pre_tokenizer.rs @@ -154,7 +154,7 @@ impl PreTokenizedString { .flat_map(|split| { split.tokens.unwrap().into_iter().map(|token| { // Replace this with the actual fields you need for the Encoding type - (token.id,String::with_capacity(0), (0, 0), None, 0) + (token.id, String::with_capacity(0), (0, 0), None, 0) }) }) .collect();