Add display capabilities to tokenizers objects

bindings/python/Cargo.toml

@@ -19,6 +19,8 @@ numpy = "0.21"
 ndarray = "0.15"
 onig = { version = "6.4", default-features = false }
 itertools = "0.12"
+derive_more = "0.99.17"
+ellipse = "0.2.0"
 
 [dependencies.tokenizers]
 path = "../../tokenizers"

bindings/python/py_src/tokenizers/__init__.pyi

@@ -725,6 +725,16 @@ class Tokenizer:
         """
         pass
 
+    @property
+    def added_tokens_decoder(self):
+        """
+        Get the underlying vocabulary
+
+        Returns:
+            :obj:`Dict[int, AddedToken]`: The vocabulary
+        """
+        pass
+
     def decode(self, ids, skip_special_tokens=True):
         """
         Decode the given list of ids back to a string
@@ -970,15 +980,6 @@ class Tokenizer:
         """
         pass
 
-    def get_added_tokens_decoder(self):
-        """
-        Get the underlying vocabulary
-
-        Returns:
-            :obj:`Dict[int, AddedToken]`: The vocabulary
-        """
-        pass
-
     def get_vocab(self, with_added_tokens=True):
         """
         Get the underlying vocabulary

bindings/python/src/decoders.rs

@@ -2,6 +2,7 @@ use std::sync::{Arc, RwLock};
 
 use crate::pre_tokenizers::from_string;
 use crate::utils::PyPattern;
+use derive_more::Display;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
@@ -28,7 +29,7 @@ use super::error::ToPyResult;
 /// This class is not supposed to be instantiated directly. Instead, any implementation of
 /// a Decoder will return an instance of this class when instantiated.
 #[pyclass(dict, module = "tokenizers.decoders", name = "Decoder", subclass)]
-#[derive(Clone, Deserialize, Serialize)]
+#[derive(Clone, Deserialize, Serialize, Display)]
 pub struct PyDecoder {
     #[serde(flatten)]
     pub(crate) decoder: PyDecoderWrapper,
@@ -114,6 +115,14 @@ impl PyDecoder {
     fn decode(&self, tokens: Vec<String>) -> PyResult<String> {
         ToPyResult(self.decoder.decode(tokens)).into()
     }
+
+    fn __str__(&self) -> PyResult<String> {
+        Ok(format!("{}", self.decoder))
+    }
+
+    fn __repr__(&self) -> PyResult<String> {
+        Ok(format!("{}", self.decoder))
+    }
 }
 
 macro_rules! getter {
@@ -478,9 +487,9 @@ impl PySequenceDecoder {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Display)]
 pub(crate) struct CustomDecoder {
-    inner: PyObject,
+    pub inner: PyObject,
 }
 
 impl CustomDecoder {
@@ -531,10 +540,12 @@ impl<'de> Deserialize<'de> for CustomDecoder {
     }
 }
 
-#[derive(Clone, Deserialize, Serialize)]
+#[derive(Clone, Deserialize, Serialize, Display)]
 #[serde(untagged)]
 pub(crate) enum PyDecoderWrapper {
+    #[display(fmt = "{}", "_0.as_ref().read().unwrap().inner")]
     Custom(Arc<RwLock<CustomDecoder>>),
+    #[display(fmt = "{}", "_0.as_ref().read().unwrap()")]
     Wrapped(Arc<RwLock<DecoderWrapper>>),
 }
 

bindings/python/src/models.rs

@@ -2,8 +2,10 @@ use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 
+use super::error::{deprecation_warning, ToPyResult};
 use crate::token::PyToken;
 use crate::trainers::PyTrainer;
+use derive_more::Display;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
@@ -16,16 +18,15 @@ use tk::models::ModelWrapper;
 use tk::{Model, Token};
 use tokenizers as tk;
 
-use super::error::{deprecation_warning, ToPyResult};
-
 /// Base class for all models
 ///
 /// The model represents the actual tokenization algorithm. This is the part that
 /// will contain and manage the learned vocabulary.
 ///
 /// This class cannot be constructed directly. Please use one of the concrete models.
 #[pyclass(module = "tokenizers.models", name = "Model", subclass)]
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone, Serialize, Deserialize, Display)]
+#[display(fmt = "{}", "model.as_ref().read().unwrap()")]
 pub struct PyModel {
     #[serde(flatten)]
     pub model: Arc<RwLock<ModelWrapper>>,
@@ -220,6 +221,12 @@ impl PyModel {
     fn get_trainer(&self, py: Python<'_>) -> PyResult<PyObject> {
         PyTrainer::from(self.model.read().unwrap().get_trainer()).get_as_subtype(py)
     }
+    fn __str__(&self) -> PyResult<String> {
+        Ok(format!("{}", self.model.read().unwrap()))
+    }
+    fn __repr__(&self) -> PyResult<String> {
+        Ok(format!("{}", self.model.read().unwrap()))
+    }
 }
 
 /// An implementation of the BPE (Byte-Pair Encoding) algorithm

bindings/python/src/normalizers.rs

@@ -1,11 +1,11 @@
 use std::sync::{Arc, RwLock};
 
+use crate::error::ToPyResult;
+use crate::utils::{PyNormalizedString, PyNormalizedStringRefMut, PyPattern};
+use derive_more::Display;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
-
-use crate::error::ToPyResult;
-use crate::utils::{PyNormalizedString, PyNormalizedStringRefMut, PyPattern};
 use serde::ser::SerializeStruct;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use tk::normalizers::{
@@ -43,7 +43,7 @@ impl PyNormalizedStringMut<'_> {
 /// This class is not supposed to be instantiated directly. Instead, any implementation of a
 /// Normalizer will return an instance of this class when instantiated.
 #[pyclass(dict, module = "tokenizers.normalizers", name = "Normalizer", subclass)]
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone, Serialize, Deserialize, Display, Debug)]
 pub struct PyNormalizer {
     #[serde(flatten)]
     pub(crate) normalizer: PyNormalizerTypeWrapper,
@@ -166,6 +166,13 @@ impl PyNormalizer {
         ToPyResult(self.normalizer.normalize(&mut normalized)).into_py()?;
         Ok(normalized.get().to_owned())
     }
+
+    fn __str__(&self) -> PyResult<String> {
+        Ok(format!("{}", self.normalizer))
+    }
+    fn __repr__(&self) -> PyResult<String> {
+        Ok(format!("{}", self.normalizer))
+    }
 }
 
 macro_rules! getter {
@@ -498,7 +505,7 @@ impl PyReplace {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Display)]
 pub(crate) struct CustomNormalizer {
     inner: PyObject,
 }
@@ -541,10 +548,12 @@ impl<'de> Deserialize<'de> for CustomNormalizer {
     }
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Display)]
 #[serde(untagged)]
 pub(crate) enum PyNormalizerWrapper {
+    #[display(fmt = "{}", "_0.inner")]
     Custom(CustomNormalizer),
+    #[display(fmt = "{}", "_0")]
     Wrapped(NormalizerWrapper),
 }
 
@@ -567,6 +576,25 @@ pub(crate) enum PyNormalizerTypeWrapper {
     Single(Arc<RwLock<PyNormalizerWrapper>>),
 }
 
+// Implement the Display trait for PyNormalizerTypeWrapper
+impl std::fmt::Display for PyNormalizerTypeWrapper {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            PyNormalizerTypeWrapper::Sequence(ref decoders) => {
+                for decoder in decoders {
+                    let decoder = decoder.read().unwrap();
+                    writeln!(f, "{}", decoder)?;
+                }
+                Ok(())
+            }
+            PyNormalizerTypeWrapper::Single(ref decoder) => {
+                let decoder = decoder.read().unwrap();
+                write!(f, "{}", decoder)
+            }
+        }
+    }
+}
+
 impl Serialize for PyNormalizerTypeWrapper {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where

bindings/python/src/pre_tokenizers.rs

@@ -23,7 +23,7 @@ use tokenizers as tk;
 
 use super::error::ToPyResult;
 use super::utils::*;
-
+use derive_more::Display;
 /// Base class for all pre-tokenizers
 ///
 /// This class is not supposed to be instantiated directly. Instead, any implementation of a
@@ -34,7 +34,7 @@ use super::utils::*;
     name = "PreTokenizer",
     subclass
 )]
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone, Serialize, Deserialize, Display)]
 pub struct PyPreTokenizer {
     #[serde(flatten)]
     pub(crate) pretok: PyPreTokenizerTypeWrapper,
@@ -181,6 +181,14 @@ impl PyPreTokenizer {
             .map(|(s, o, _)| (s.to_owned(), o))
             .collect())
     }
+
+    fn __str__(&self) -> PyResult<String> {
+        Ok(format!("{}", self.pretok))
+    }
+
+    fn __repr__(&self) -> PyResult<String> {
+        Ok(format!("{}", self.pretok))
+    }
 }
 
 macro_rules! getter {
@@ -587,7 +595,7 @@ impl PyUnicodeScripts {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Display)]
 pub(crate) struct CustomPreTokenizer {
     inner: PyObject,
 }
@@ -631,7 +639,7 @@ impl<'de> Deserialize<'de> for CustomPreTokenizer {
     }
 }
 
-#[derive(Clone, Deserialize)]
+#[derive(Clone, Deserialize, Display)]
 #[serde(untagged)]
 pub(crate) enum PyPreTokenizerWrapper {
     Custom(CustomPreTokenizer),
@@ -650,10 +658,23 @@ impl Serialize for PyPreTokenizerWrapper {
     }
 }
 
-#[derive(Clone, Deserialize)]
+#[derive(Clone, Deserialize, Display)]
 #[serde(untagged)]
 pub(crate) enum PyPreTokenizerTypeWrapper {
+    #[display(
+        fmt = "[{}]",
+        "_0.iter()
+    .map(|d| d.as_ref().read().unwrap().to_string())
+    .fold(String::new(), |mut acc, s| {
+        if !acc.is_empty() {
+            acc.push_str(\", \");
+        }
+        acc.push_str(&s);
+        acc
+    })"
+    )]
     Sequence(Vec<Arc<RwLock<PyPreTokenizerWrapper>>>),
+    #[display(fmt = "{}", "_0.as_ref().read().unwrap()")]
     Single(Arc<RwLock<PyPreTokenizerWrapper>>),
 }
 

bindings/python/src/processors.rs

@@ -1,12 +1,12 @@
 use std::convert::TryInto;
 use std::sync::Arc;
 
+use crate::encoding::PyEncoding;
+use crate::error::ToPyResult;
+use derive_more::Display;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
-
-use crate::encoding::PyEncoding;
-use crate::error::ToPyResult;
 use serde::{Deserialize, Serialize};
 use tk::processors::bert::BertProcessing;
 use tk::processors::byte_level::ByteLevel;
@@ -27,7 +27,7 @@ use tokenizers as tk;
     name = "PostProcessor",
     subclass
 )]
-#[derive(Clone, Deserialize, Serialize)]
+#[derive(Clone, Deserialize, Serialize, Display)]
 pub struct PyPostProcessor {
     #[serde(flatten)]
     pub processor: Arc<PostProcessorWrapper>,
@@ -139,6 +139,14 @@ impl PyPostProcessor {
         .into_py()?;
         Ok(final_encoding.into())
     }
+
+    fn __str__(&self) -> PyResult<String> {
+        Ok(format!("{}", &self))
+    }
+
+    fn __repr__(&self) -> PyResult<String>{
+        Ok(format!("{}", &self))
+    }
 }
 
 /// This post-processor takes care of adding the special tokens needed by