some finishing touch

huggingface · Jun 10, 2024 · fe95add · fe95add
1 parent e0d35e0
commit fe95add
Show file tree

Hide file tree

Showing 4 changed files with 1 addition and 4 deletions.
diff --git a/bindings/python/src/decoders.rs b/bindings/python/src/decoders.rs
@@ -30,7 +30,6 @@ use super::error::ToPyResult;
 /// a Decoder will return an instance of this class when instantiated.
 #[pyclass(dict, module = "tokenizers.decoders", name = "Decoder", subclass)]
 #[derive(Clone, Deserialize, Serialize, Display)]
-#[display(fmt = "{}", decoder)]
 pub struct PyDecoder {
     #[serde(flatten)]
     pub(crate) decoder: PyDecoderWrapper,

diff --git a/bindings/python/src/normalizers.rs b/bindings/python/src/normalizers.rs
@@ -44,7 +44,6 @@ impl PyNormalizedStringMut<'_> {
 /// Normalizer will return an instance of this class when instantiated.
 #[pyclass(dict, module = "tokenizers.normalizers", name = "Normalizer", subclass)]
 #[derive(Clone, Serialize, Deserialize, Display, Debug)]
-#[display(fmt = "{}", normalizer)]
 pub struct PyNormalizer {
     #[serde(flatten)]
     pub(crate) normalizer: PyNormalizerTypeWrapper,

diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs
@@ -35,7 +35,6 @@ use derive_more::Display;
     subclass
 )]
 #[derive(Clone, Serialize, Deserialize, Display)]
-#[display(fmt = "PreTokenizer(pretok={})", pretok)]
 pub struct PyPreTokenizer {
     #[serde(flatten)]
     pub(crate) pretok: PyPreTokenizerTypeWrapper,

diff --git a/tokenizers/src/pre_tokenizers/sequence.rs b/tokenizers/src/pre_tokenizers/sequence.rs
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
 #[macro_rules_attribute(impl_serde_type!)]
 #[derive(Clone, Debug, PartialEq, Display)]
 #[display(
-    fmt = "pre_tokenizers.Seqence([{}])",
+    fmt = "Seqence([{}])",
     "pretokenizers.iter().fold(String::new(), |mut acc, p| {
     if !acc.is_empty(){
        acc.push_str(\", \")