Update bindings/python/src/tokenizer.rs

Co-authored-by: Nicolas Patry <[email protected]>
huggingface · Nov 5, 2024 · e1169ee · e1169ee
1 parent 8c96f47
commit e1169ee
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs
@@ -1036,7 +1036,7 @@ impl PyTokenizer {
         is_pretokenized: bool,
         add_special_tokens: bool,
     ) -> PyResult<Vec<PyEncoding>> {
-        let mut items = Vec::<tk::EncodeInput>::new();
+        let mut items = Vec::<tk::EncodeInput>::with_capacity(input.len());
         for i in 0..input.len()? {
             let item = input.get_item(i)?;
             let item: tk::EncodeInput = if is_pretokenized {