__repr__ should use Debug?

bindings/python/src/decoders.rs

@@ -29,7 +29,7 @@ use super::error::ToPyResult;
 /// This class is not supposed to be instantiated directly. Instead, any implementation of
 /// a Decoder will return an instance of this class when instantiated.
 #[pyclass(dict, module = "tokenizers.decoders", name = "Decoder", subclass)]
-#[derive(Clone, Deserialize, Serialize, Display)]
+#[derive(Clone, Deserialize, Serialize, Display, Debug)]
 #[display(fmt = "{}", decoder)]
 pub struct PyDecoder {
     #[serde(flatten)]
@@ -488,7 +488,7 @@ impl PySequenceDecoder {
     }
 }
 
-#[derive(Clone, Display)]
+#[derive(Clone, Display, Debug)]
 pub(crate) struct CustomDecoder {
     pub inner: PyObject,
 }
@@ -541,7 +541,7 @@ impl<'de> Deserialize<'de> for CustomDecoder {
     }
 }
 
-#[derive(Clone, Deserialize, Serialize, Display)]
+#[derive(Clone, Deserialize, Serialize, Display, Debug)]
 #[serde(untagged)]
 pub(crate) enum PyDecoderWrapper {
     #[display(fmt = "{}", "_0.as_ref().read().unwrap().inner")]

bindings/python/src/models.rs

@@ -25,7 +25,7 @@ use tokenizers as tk;
 ///
 /// This class cannot be constructed directly. Please use one of the concrete models.
 #[pyclass(module = "tokenizers.models", name = "Model", subclass)]
-#[derive(Clone, Serialize, Deserialize, Display)]
+#[derive(Clone, Serialize, Deserialize, Display, Debug)]
 #[display(fmt = "{}", "model.as_ref().read().unwrap()")]
 pub struct PyModel {
     #[serde(flatten)]

bindings/python/src/normalizers.rs

@@ -90,7 +90,6 @@ impl PyNormalizer {
             },
         })
     }
-
 }
 
 impl Normalizer for PyNormalizer {
@@ -571,33 +570,42 @@ impl Serialize for PyNormalizerWrapper {
     }
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Display)]
 #[serde(untagged)]
 pub(crate) enum PyNormalizerTypeWrapper {
+    #[display(fmt = "Normalizer.Sequence([{}])", "_0.iter()
+    .map(|d| d.as_ref().read().unwrap().to_string())
+    .fold(String::new(), |mut acc, s| {
+        if !acc.is_empty() {
+            acc.push_str(\", \");
+        }
+        acc.push_str(&s);
+        acc
+    })")]
     Sequence(Vec<Arc<RwLock<PyNormalizerWrapper>>>),
+    #[display(fmt ="Normalizer.{}", "_0.as_ref().read().unwrap()")]
     Single(Arc<RwLock<PyNormalizerWrapper>>),
 }
 
-// Implement the Display trait for PyNormalizerTypeWrapper
-impl std::fmt::Display for PyNormalizerTypeWrapper {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        match self {
-            PyNormalizerTypeWrapper::Sequence(ref decoders) => {
-                for decoder in decoders {
-                    let decoder = decoder.read().unwrap();
-                    writeln!(f, "{}", decoder)?;
+// // Implement the Display trait for PyNormalizerTypeWrapper
+// impl std::fmt::Display for PyNormalizerTypeWrapper {
+//     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+//         match self {
+//             PyNormalizerTypeWrapper::Sequence(ref decoders) => {
+//                 for decoder in decoders {
+//                     let decoder = decoder.read().unwrap();
+//                     writeln!(f, "{}", decoder)?;
 
-                }
-            writeln!(f, "?????")?; 
-            Ok(())
-            }
-            PyNormalizerTypeWrapper::Single(ref decoder) => {
-                let decoder = decoder.read().unwrap();
-                write!(f, "{}", decoder)
-            }
-        }
-    }
-}
+//                 }
+//             Ok(())
+//             }
+//             PyNormalizerTypeWrapper::Single(ref decoder) => {
+//                 let decoder = decoder.read().unwrap();
+//                 write!(f, "{}", decoder)
+//             }
+//         }
+//     }
+// }
 
 impl Serialize for PyNormalizerTypeWrapper {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

bindings/python/src/pre_tokenizers.rs

@@ -34,8 +34,7 @@ use derive_more::Display;
     name = "PreTokenizer",
     subclass
 )]
-#[derive(Clone, Serialize, Deserialize, Display)]
-#[display(fmt = "PreTokenizer(pretok={})", pretok)]
+#[derive(Clone, Serialize, Deserialize, Display, Debug)]
 pub struct PyPreTokenizer {
     #[serde(flatten)]
     pub(crate) pretok: PyPreTokenizerTypeWrapper,
@@ -596,7 +595,7 @@ impl PyUnicodeScripts {
     }
 }
 
-#[derive(Clone, Display)]
+#[derive(Clone, Display, Debug)]
 pub(crate) struct CustomPreTokenizer {
     inner: PyObject,
 }
@@ -640,7 +639,7 @@ impl<'de> Deserialize<'de> for CustomPreTokenizer {
     }
 }
 
-#[derive(Clone, Deserialize, Display)]
+#[derive(Clone, Deserialize, Display, Debug)]
 #[display(fmt="{}")]
 #[serde(untagged)]
 pub(crate) enum PyPreTokenizerWrapper {
@@ -660,21 +659,21 @@ impl Serialize for PyPreTokenizerWrapper {
     }
 }
 
-#[derive(Clone, Deserialize, Display)]
+#[derive(Clone, Deserialize, Display, Debug)]
 #[serde(untagged)]
 pub(crate) enum PyPreTokenizerTypeWrapper {
-    #[display(fmt = "[{}]", "_0.iter()
+    #[display(fmt = "PreTokenizer.Sequence([{}])", "_0.iter()
     .map(|d| d.as_ref().read().unwrap().to_string())
     .fold(String::new(), |mut acc, s| {
         if !acc.is_empty() {
             acc.push_str(\", \");
         }
         acc.push_str(&s);
         acc
-    })")]
+    })")] // This one is only used when the pre_tokenizer is set in python
     Sequence(Vec<Arc<RwLock<PyPreTokenizerWrapper>>>),
     #[display(fmt ="{}", "_0.as_ref().read().unwrap()")]
-    Single(Arc<RwLock<PyPreTokenizerWrapper>>),
+    Single(Arc<RwLock<PyPreTokenizerWrapper>>), // this one can actually be a sequence in rust
 }
 
 impl Serialize for PyPreTokenizerTypeWrapper {

bindings/python/src/processors.rs

@@ -27,7 +27,7 @@ use tokenizers as tk;
     name = "PostProcessor",
     subclass
 )]
-#[derive(Clone, Deserialize, Serialize, Display)]
+#[derive(Clone, Deserialize, Serialize, Display, Debug)]
 pub struct PyPostProcessor {
     #[serde(flatten)]
     pub processor: Arc<PostProcessorWrapper>,

bindings/python/src/tokenizer.rs

@@ -1416,7 +1416,7 @@ impl PyTokenizer {
     }
 
     fn __repr__(&self) -> PyResult<String>{
-        Ok(format!("{}", self.tokenizer))
+        Ok(format!("{:?}", self.tokenizer))
     }
 }