From e50ee91045d29993c49ddf324b43391bef954966 Mon Sep 17 00:00:00 2001
From: Mike Lui <mikelui@meta.com>
Date: Sat, 11 Nov 2023 00:48:53 -0500
Subject: [PATCH] pyo3: update to 0.20

---
 bindings/python/Cargo.toml             | 6 +++---
 bindings/python/src/processors.rs      | 6 +++---
 bindings/python/src/tokenizer.rs       | 1 -
 bindings/python/src/utils/iterators.rs | 1 -
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index cf9cd5a0f..86446441d 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -14,8 +14,8 @@ serde = { version = "1.0", features = [ "rc", "derive" ]}
 serde_json = "1.0"
 libc = "0.2"
 env_logger = "0.10.0"
-pyo3 = { version = "0.19" }
-numpy = "0.19.0"
+pyo3 = { version = "0.20" }
+numpy = "0.20.0"
 ndarray = "0.15"
 onig = { version = "6.4", default-features = false }
 itertools = "0.11"
@@ -26,7 +26,7 @@ path = "../../tokenizers"
 
 [dev-dependencies]
 tempfile = "3.8"
-pyo3 = { version = "0.19", features = ["auto-initialize"] }
+pyo3 = { version = "0.20", features = ["auto-initialize"] }
 
 [features]
 defaut = ["pyo3/extension-module"]
diff --git a/bindings/python/src/processors.rs b/bindings/python/src/processors.rs
index 3600deec5..55a69287e 100644
--- a/bindings/python/src/processors.rs
+++ b/bindings/python/src/processors.rs
@@ -266,15 +266,15 @@ impl FromPyObject<'_> for PySpecialToken {
             Ok(Self(v.into()))
         } else if let Ok(d) = ob.downcast::<PyDict>() {
             let id = d
-                .get_item("id")
+                .get_item("id")?
                 .ok_or_else(|| exceptions::PyValueError::new_err("`id` must be specified"))?
                 .extract::<String>()?;
             let ids = d
-                .get_item("ids")
+                .get_item("ids")?
                 .ok_or_else(|| exceptions::PyValueError::new_err("`ids` must be specified"))?
                 .extract::<Vec<u32>>()?;
             let tokens = d
-                .get_item("tokens")
+                .get_item("tokens")?
                 .ok_or_else(|| exceptions::PyValueError::new_err("`tokens` must be specified"))?
                 .extract::<Vec<String>>()?;
 
diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs
index 25e93159d..77e071314 100644
--- a/bindings/python/src/tokenizer.rs
+++ b/bindings/python/src/tokenizer.rs
@@ -7,7 +7,6 @@ use pyo3::exceptions;
 use pyo3::intern;
 use pyo3::prelude::*;
 use pyo3::types::*;
-use pyo3::AsPyPointer;
 use tk::models::bpe::BPE;
 use tk::tokenizer::{
     Model, PaddingDirection, PaddingParams, PaddingStrategy, PostProcessor, TokenizerImpl,
diff --git a/bindings/python/src/utils/iterators.rs b/bindings/python/src/utils/iterators.rs
index cf6310b4d..d19a54eb5 100644
--- a/bindings/python/src/utils/iterators.rs
+++ b/bindings/python/src/utils/iterators.rs
@@ -1,5 +1,4 @@
 use pyo3::prelude::*;
-use pyo3::AsPyPointer;
 use std::collections::VecDeque;
 
 /// An simple iterator that can be instantiated with a specified length.