Upgraded everything.

bindings/python/src/decoders.rs

@@ -84,7 +84,7 @@ impl PyDecoder {
                 e
             ))
         })?;
-        Ok(PyBytes::new(py, data.as_bytes()).to_object(py))
+        Ok(PyBytes::new_bound(py, data.as_bytes()).to_object(py))
     }
 
     fn __setstate__(&mut self, py: Python, state: PyObject) -> PyResult<()> {
@@ -160,7 +160,7 @@ pub struct PyByteLevelDec {}
 impl PyByteLevelDec {
     #[new]
     #[pyo3(signature = (**_kwargs), text_signature = "(self)")]
-    fn new(_kwargs: Option<&PyDict>) -> (Self, PyDecoder) {
+    fn new(_kwargs: Option<&Bound<'_, PyDict>>) -> (Self, PyDecoder) {
         (PyByteLevelDec {}, ByteLevel::default().into())
     }
 }
@@ -462,7 +462,7 @@ pub struct PySequenceDecoder {}
 impl PySequenceDecoder {
     #[new]
     #[pyo3(signature = (decoders_py), text_signature = "(self, decoders)")]
-    fn new(decoders_py: &PyList) -> PyResult<(Self, PyDecoder)> {
+    fn new(decoders_py: &Bound<'_, PyList>) -> PyResult<(Self, PyDecoder)> {
         let mut decoders: Vec<DecoderWrapper> = Vec::with_capacity(decoders_py.len());
         for decoder_py in decoders_py.iter() {
             let decoder: PyRef<PyDecoder> = decoder_py.extract()?;
@@ -475,8 +475,8 @@ impl PySequenceDecoder {
         Ok((PySequenceDecoder {}, Sequence::new(decoders).into()))
     }
 
-    fn __getnewargs__<'p>(&self, py: Python<'p>) -> &'p PyTuple {
-        PyTuple::new(py, [PyList::empty(py)])
+    fn __getnewargs__<'p>(&self, py: Python<'p>) -> Bound<'p, PyTuple> {
+        PyTuple::new_bound(py, [PyList::empty_bound(py)])
     }
 }
 
@@ -496,7 +496,7 @@ impl Decoder for CustomDecoder {
         Python::with_gil(|py| {
             let decoded = self
                 .inner
-                .call_method(py, "decode", (tokens,), None)?
+                .call_method_bound(py, "decode", (tokens,), None)?
                 .extract(py)?;
             Ok(decoded)
         })
@@ -506,7 +506,7 @@ impl Decoder for CustomDecoder {
         Python::with_gil(|py| {
             let decoded = self
                 .inner
-                .call_method(py, "decode_chain", (tokens,), None)?
+                .call_method_bound(py, "decode_chain", (tokens,), None)?
                 .extract(py)?;
             Ok(decoded)
         })
@@ -571,7 +571,7 @@ impl Decoder for PyDecoderWrapper {
 
 /// Decoders Module
 #[pymodule]
-pub fn decoders(_py: Python, m: &PyModule) -> PyResult<()> {
+pub fn decoders(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyDecoder>()?;
     m.add_class::<PyByteLevelDec>()?;
     m.add_class::<PyReplaceDec>()?;
@@ -601,7 +601,7 @@ mod test {
         Python::with_gil(|py| {
             let py_dec = PyDecoder::new(Metaspace::default().into());
             let py_meta = py_dec.get_as_subtype(py).unwrap();
-            assert_eq!("Metaspace", py_meta.as_ref(py).get_type().qualname().unwrap());
+            assert_eq!("Metaspace", py_meta.bind(py).get_type().qualname().unwrap());
         })
     }
 

bindings/python/src/encoding.rs

@@ -37,7 +37,7 @@ impl PyEncoding {
                 e
             ))
         })?;
-        Ok(PyBytes::new(py, data.as_bytes()).to_object(py))
+        Ok(PyBytes::new_bound(py, data.as_bytes()).to_object(py))
     }
 
     fn __setstate__(&mut self, py: Python, state: PyObject) -> PyResult<()> {
@@ -391,10 +391,10 @@ impl PyEncoding {
     #[pyo3(
         text_signature = "(self, length, direction='right', pad_id=0, pad_type_id=0, pad_token='[PAD]')"
     )]
-    fn pad(&mut self, length: usize, kwargs: Option<&PyDict>) -> PyResult<()> {
+    fn pad(&mut self, length: usize, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<()> {
         let mut pad_id = 0;
         let mut pad_type_id = 0;
-        let mut pad_token = "[PAD]";
+        let mut pad_token = "[PAD]".to_string();
         let mut direction = PaddingDirection::Right;
 
         if let Some(kwargs) = kwargs {
@@ -422,7 +422,7 @@ impl PyEncoding {
             }
         }
         self.encoding
-            .pad(length, pad_id, pad_type_id, pad_token, direction);
+            .pad(length, pad_id, pad_type_id, &pad_token, direction);
         Ok(())
     }
 

bindings/python/src/error.rs

@@ -35,7 +35,7 @@ impl<T> ToPyResult<T> {
 }
 
 pub(crate) fn deprecation_warning(py: Python<'_>, version: &str, message: &str) -> PyResult<()> {
-    let deprecation_warning = py.import("builtins")?.getattr("DeprecationWarning")?;
+    let deprecation_warning = py.import_bound("builtins")?.getattr("DeprecationWarning")?;
     let full_message = format!("Deprecated in {}: {}", version, message);
-    pyo3::PyErr::warn(py, deprecation_warning, &full_message, 0)
+    pyo3::PyErr::warn_bound(py, &deprecation_warning, &full_message, 0)
 }

bindings/python/src/lib.rs

@@ -47,7 +47,7 @@ extern "C" fn child_after_fork() {
 
 /// Tokenizers Module
 #[pymodule]
-pub fn tokenizers(_py: Python, m: &PyModule) -> PyResult<()> {
+pub fn tokenizers(m: &Bound<'_ , PyModule>) -> PyResult<()> {
     let _ = env_logger::try_init_from_env("TOKENIZERS_LOG");
 
     // Register the fork callback

bindings/python/src/models.rs

@@ -105,7 +105,7 @@ impl PyModel {
                 e
             ))
         })?;
-        Ok(PyBytes::new(py, data.as_bytes()).to_object(py))
+        Ok(PyBytes::new_bound(py, data.as_bytes()).to_object(py))
     }
 
     fn __setstate__(&mut self, py: Python, state: PyObject) -> PyResult<()> {
@@ -260,7 +260,7 @@ impl PyModel {
 pub struct PyBPE {}
 
 impl PyBPE {
-    fn with_builder(mut builder: BpeBuilder, kwargs: Option<&PyDict>) -> PyResult<(Self, PyModel)> {
+    fn with_builder(mut builder: BpeBuilder, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<(Self, PyModel)> {
         if let Some(kwargs) = kwargs {
             for (key, value) in kwargs {
                 let key: &str = key.extract()?;
@@ -417,7 +417,7 @@ impl PyBPE {
         py: Python<'_>,
         vocab: Option<PyVocab>,
         merges: Option<PyMerges>,
-        kwargs: Option<&PyDict>,
+        kwargs: Option<&Bound<'_, PyDict>>,
     ) -> PyResult<(Self, PyModel)> {
         if (vocab.is_some() && merges.is_none()) || (vocab.is_none() && merges.is_some()) {
             return Err(exceptions::PyValueError::new_err(
@@ -502,11 +502,11 @@ impl PyBPE {
     #[pyo3(signature = (vocab, merges, **kwargs))]
     #[pyo3(text_signature = "(cls, vocab, merge, **kwargs)")]
     fn from_file(
-        _cls: &PyType,
+        _cls: &Bound<'_, PyType>,
         py: Python,
         vocab: &str,
         merges: &str,
-        kwargs: Option<&PyDict>,
+        kwargs: Option<&Bound<'_, PyDict>>,
     ) -> PyResult<Py<Self>> {
         let (vocab, merges) = BPE::read_file(vocab, merges).map_err(|e| {
             exceptions::PyException::new_err(format!("Error while reading BPE files: {}", e))
@@ -540,7 +540,7 @@ pub struct PyWordPiece {}
 impl PyWordPiece {
     fn with_builder(
         mut builder: WordPieceBuilder,
-        kwargs: Option<&PyDict>,
+        kwargs: Option<&Bound<'_, PyDict>>,
     ) -> PyResult<(Self, PyModel)> {
         if let Some(kwargs) = kwargs {
             for (key, val) in kwargs {
@@ -612,7 +612,7 @@ impl PyWordPiece {
     fn new(
         py: Python<'_>,
         vocab: Option<PyVocab>,
-        kwargs: Option<&PyDict>,
+        kwargs: Option<&Bound<'_, PyDict>>,
     ) -> PyResult<(Self, PyModel)> {
         let mut builder = WordPiece::builder();
 
@@ -677,10 +677,10 @@ impl PyWordPiece {
     #[pyo3(signature = (vocab, **kwargs))]
     #[pyo3(text_signature = "(vocab, **kwargs)")]
     fn from_file(
-        _cls: &PyType,
+        _cls: &Bound<'_, PyType>,
         py: Python,
         vocab: &str,
-        kwargs: Option<&PyDict>,
+        kwargs: Option<&Bound<'_, PyDict>>,
     ) -> PyResult<Py<Self>> {
         let vocab = WordPiece::read_file(vocab).map_err(|e| {
             exceptions::PyException::new_err(format!("Error while reading WordPiece file: {}", e))
@@ -796,7 +796,7 @@ impl PyWordLevel {
     #[pyo3(signature = (vocab, unk_token = None))]
     #[pyo3(text_signature = "(vocab, unk_token)")]
     fn from_file(
-        _cls: &PyType,
+        _cls: &Bound<'_, PyType>,
         py: Python,
         vocab: &str,
         unk_token: Option<String>,
@@ -849,7 +849,7 @@ impl PyUnigram {
 
 /// Models Module
 #[pymodule]
-pub fn models(_py: Python, m: &PyModule) -> PyResult<()> {
+pub fn models(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyModel>()?;
     m.add_class::<PyBPE>()?;
     m.add_class::<PyWordPiece>()?;
@@ -870,7 +870,7 @@ mod test {
         Python::with_gil(|py| {
             let py_model = PyModel::from(BPE::default());
             let py_bpe = py_model.get_as_subtype(py).unwrap();
-            assert_eq!("BPE", py_bpe.as_ref(py).get_type().qualname().unwrap());
+            assert_eq!("BPE", py_bpe.bind(py).get_type().qualname().unwrap());
         })
     }
 

bindings/python/src/normalizers.rs

@@ -113,7 +113,7 @@ impl PyNormalizer {
                 e
             ))
         })?;
-        Ok(PyBytes::new(py, data.as_bytes()).to_object(py))
+        Ok(PyBytes::new_bound(py, data.as_bytes()).to_object(py))
     }
 
     fn __setstate__(&mut self, py: Python, state: PyObject) -> PyResult<()> {
@@ -345,7 +345,7 @@ pub struct PySequence {}
 impl PySequence {
     #[new]
     #[pyo3(text_signature = None)]
-    fn new(normalizers: &PyList) -> PyResult<(Self, PyNormalizer)> {
+    fn new(normalizers: &Bound<'_, PyList>) -> PyResult<(Self, PyNormalizer)> {
         let mut sequence = Vec::with_capacity(normalizers.len());
         for n in normalizers.iter() {
             let normalizer: PyRef<PyNormalizer> = n.extract()?;
@@ -360,8 +360,8 @@ impl PySequence {
         ))
     }
 
-    fn __getnewargs__<'p>(&self, py: Python<'p>) -> &'p PyTuple {
-        PyTuple::new(py, [PyList::empty(py)])
+    fn __getnewargs__<'p>(&self, py: Python<'p>) -> Bound<'p, PyTuple> {
+        PyTuple::new_bound(py, [PyList::empty_bound(py)])
     }
 
     fn __len__(&self) -> usize {
@@ -467,8 +467,8 @@ pub struct PyPrecompiled {}
 impl PyPrecompiled {
     #[new]
     #[pyo3(text_signature = "(self, precompiled_charsmap)")]
-    fn new(py_precompiled_charsmap: &PyBytes) -> PyResult<(Self, PyNormalizer)> {
-        let precompiled_charsmap: Vec<u8> = FromPyObject::extract(py_precompiled_charsmap)?;
+    fn new(precompiled_charsmap: Vec<u8>) -> PyResult<(Self, PyNormalizer)> {
+        // let precompiled_charsmap: Vec<u8> = FromPyObject::extract(py_precompiled_charsmap)?;
         Ok((
             PyPrecompiled {},
             Precompiled::from(&precompiled_charsmap)
@@ -512,7 +512,7 @@ impl tk::tokenizer::Normalizer for CustomNormalizer {
     fn normalize(&self, normalized: &mut NormalizedString) -> tk::Result<()> {
         Python::with_gil(|py| {
             let normalized = PyNormalizedStringRefMut::new(normalized);
-            let py_normalized = self.inner.as_ref(py);
+            let py_normalized = self.inner.bind(py);
             py_normalized.call_method("normalize", (normalized.get(),), None)?;
             Ok(())
         })
@@ -635,7 +635,7 @@ impl Normalizer for PyNormalizerWrapper {
 
 /// Normalizers Module
 #[pymodule]
-pub fn normalizers(_py: Python, m: &PyModule) -> PyResult<()> {
+pub fn normalizers(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyNormalizer>()?;
     m.add_class::<PyBertNormalizer>()?;
     m.add_class::<PyNFD>()?;
@@ -667,7 +667,7 @@ mod test {
         Python::with_gil(|py| {
             let py_norm = PyNormalizer::new(NFC.into());
             let py_nfc = py_norm.get_as_subtype(py).unwrap();
-            assert_eq!("NFC", py_nfc.as_ref(py).get_type().qualname().unwrap());
+            assert_eq!("NFC", py_nfc.bind(py).get_type().qualname().unwrap());
         })
     }
 

bindings/python/src/pre_tokenizers.rs

@@ -118,7 +118,7 @@ impl PyPreTokenizer {
                 e
             ))
         })?;
-        Ok(PyBytes::new(py, data.as_bytes()).to_object(py))
+        Ok(PyBytes::new_bound(py, data.as_bytes()).to_object(py))
     }
 
     fn __setstate__(&mut self, py: Python, state: PyObject) -> PyResult<()> {
@@ -263,7 +263,7 @@ impl PyByteLevel {
     fn new(
         add_prefix_space: bool,
         use_regex: bool,
-        _kwargs: Option<&PyDict>,
+        _kwargs: Option<&Bound<'_, PyDict>>,
     ) -> (Self, PyPreTokenizer) {
         (
             PyByteLevel {},
@@ -352,8 +352,8 @@ impl PySplit {
         ))
     }
 
-    fn __getnewargs__<'p>(&self, py: Python<'p>) -> &'p PyTuple {
-        PyTuple::new(py, [" ", "removed"])
+    fn __getnewargs__<'p>(&self, py: Python<'p>) -> Bound<'p, PyTuple> {
+        PyTuple::new_bound(py, [" ", "removed"])
     }
 }
 
@@ -385,8 +385,8 @@ impl PyCharDelimiterSplit {
         ))
     }
 
-    fn __getnewargs__<'p>(&self, py: Python<'p>) -> &'p PyTuple {
-        PyTuple::new(py, [" "])
+    fn __getnewargs__<'p>(&self, py: Python<'p>) -> Bound<'p, PyTuple> {
+        PyTuple::new_bound(py, [" "])
     }
 }
 
@@ -430,7 +430,7 @@ pub struct PySequence {}
 impl PySequence {
     #[new]
     #[pyo3(text_signature = "(self, pretokenizers)")]
-    fn new(pre_tokenizers: &PyList) -> PyResult<(Self, PyPreTokenizer)> {
+    fn new(pre_tokenizers: &Bound<'_, PyList>) -> PyResult<(Self, PyPreTokenizer)> {
         let mut sequence = Vec::with_capacity(pre_tokenizers.len());
         for n in pre_tokenizers.iter() {
             let pretokenizer: PyRef<PyPreTokenizer> = n.extract()?;
@@ -447,8 +447,8 @@ impl PySequence {
         ))
     }
 
-    fn __getnewargs__<'p>(&self, py: Python<'p>) -> &'p PyTuple {
-        PyTuple::new(py, [PyList::empty(py)])
+    fn __getnewargs__<'p>(&self, py: Python<'p>) -> Bound<'p, PyTuple> {
+        PyTuple::new_bound(py, [PyList::empty_bound(py)])
     }
 }
 
@@ -599,7 +599,7 @@ impl tk::tokenizer::PreTokenizer for CustomPreTokenizer {
     fn pre_tokenize(&self, sentence: &mut PreTokenizedString) -> tk::Result<()> {
         Python::with_gil(|py| {
             let pretok = PyPreTokenizedStringRefMut::new(sentence);
-            let py_pretok = self.inner.as_ref(py);
+            let py_pretok = self.inner.bind(py);
             py_pretok.call_method("pre_tokenize", (pretok.get(),), None)?;
             Ok(())
         })
@@ -722,7 +722,7 @@ impl PreTokenizer for PyPreTokenizerWrapper {
 
 /// PreTokenizers Module
 #[pymodule]
-pub fn pre_tokenizers(_py: Python, m: &PyModule) -> PyResult<()> {
+pub fn pre_tokenizers(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyPreTokenizer>()?;
     m.add_class::<PyByteLevel>()?;
     m.add_class::<PyWhitespace>()?;
@@ -754,7 +754,7 @@ mod test {
         Python::with_gil(|py| {
             let py_norm = PyPreTokenizer::new(Whitespace {}.into());
             let py_wsp = py_norm.get_as_subtype(py).unwrap();
-            assert_eq!("Whitespace", py_wsp.as_ref(py).get_type().qualname().unwrap());
+            assert_eq!("Whitespace", py_wsp.bind(py).get_type().qualname().unwrap());
         })
     }