diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index a4ec7b26f..45dfa09e7 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -1036,7 +1036,7 @@ impl PyTokenizer { is_pretokenized: bool, add_special_tokens: bool, ) -> PyResult> { - let mut items = Vec::::new(); + let mut items = Vec::::with_capacity(input.len()); for i in 0..input.len()? { let item = input.get_item(i)?; let item: tk::EncodeInput = if is_pretokenized {