update sequences and basic enums to show xxxx.Sequence

huggingface · Jun 10, 2024 · 9c45e8f · 9c45e8f
1 parent 15f877e
commit 9c45e8f
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 1 deletion.
diff --git a/bindings/python/src/processors.rs b/bindings/python/src/processors.rs
@@ -143,6 +143,10 @@ impl PyPostProcessor {
     fn __str__(&self) -> PyResult<String> {
         Ok(format!("{}", &self))
     }
+
+    fn __repr__(&self) -> PyResult<String>{
+        Ok(format!("{}", &self))
+    }
 }
 
 /// This post-processor takes care of adding the special tokens needed by

diff --git a/tokenizers/src/decoders/mod.rs b/tokenizers/src/decoders/mod.rs
@@ -25,6 +25,7 @@ use derive_more::Display;
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Clone, Debug, Display)]
+#[display(fmt="decoders.{})]
 #[serde(untagged)]
 pub enum DecoderWrapper {
     BPE(BPEDecoder),

diff --git a/tokenizers/src/decoders/sequence.rs b/tokenizers/src/decoders/sequence.rs
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
 #[macro_rules_attribute(impl_serde_type!)]
 #[derive(Clone, Debug, Display)]
 #[display(
-    fmt = "decoders.Sequence([{}])",
+    fmt = "Sequence([{}])",
     "decoders.iter().map(|d| d.to_string()).fold( String::new(), |mut acc, s|{
         if !acc.is_empty(){
             acc.push_str(\", \");

diff --git a/tokenizers/src/pre_tokenizers/mod.rs b/tokenizers/src/pre_tokenizers/mod.rs
@@ -25,6 +25,7 @@ use crate::{PreTokenizedString, PreTokenizer};
 use derive_more::Display;
 
 #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Display)]
+#[display(fmt="pre_tokenizers.{}")]
 #[serde(untagged)]
 pub enum PreTokenizerWrapper {
     BertPreTokenizer(BertPreTokenizer),