fix with capacity

huggingface · Nov 5, 2024 · 6056e0c · 6056e0c
1 parent e1169ee
commit 6056e0c
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs
@@ -1036,7 +1036,7 @@ impl PyTokenizer {
         is_pretokenized: bool,
         add_special_tokens: bool,
     ) -> PyResult<Vec<PyEncoding>> {
-        let mut items = Vec::<tk::EncodeInput>::with_capacity(input.len());
+        let mut items = Vec::<tk::EncodeInput>::with_capacity(input.len()?);
         for i in 0..input.len()? {
             let item = input.get_item(i)?;
             let item: tk::EncodeInput = if is_pretokenized {
@@ -1096,7 +1096,7 @@ impl PyTokenizer {
         is_pretokenized: bool,
         add_special_tokens: bool,
     ) -> PyResult<Vec<PyEncoding>> {
-        let mut items = Vec::<tk::EncodeInput>::new();
+        let mut items = Vec::<tk::EncodeInput>::with_capacity(input.len()?);
         for i in 0..input.len()? {
             let item = input.get_item(i)?;
             let item: tk::EncodeInput = if is_pretokenized {