update display derive

huggingface · Jun 12, 2024 · 4c3f37a · 4c3f37a
1 parent f67af9c
commit 4c3f37a
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 35 deletions.
diff --git a/tokenizers/display_derive/Cargo.toml b/tokenizers/display_derive/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 syn = "1.0"
 quote = "1.0"
 proc-macro2 = "1.0"
+unicode-xid = "0.2.4"
 
 [lib]
 proc-macro = true
diff --git a/tokenizers/display_derive/src/lib.rs b/tokenizers/display_derive/src/lib.rs
@@ -1,12 +1,13 @@
 extern crate proc_macro;
 use proc_macro::TokenStream;
 use quote::{format_ident,quote};
-use syn::{parse_macro_input, Data, DeriveInput, Fields, Lit, Meta, MetaList, NestedMeta};
+use syn::{parse_macro_input, DeriveInput}; 
 mod vendored;
+mod parsing;
 use vendored::FmtAttribute;
 
 #[proc_macro_derive(Display)]
-pub fn display_derive(input: TokenStream) -> syn::Result<TokenStream>  {
+pub fn display_derive(input: TokenStream) -> TokenStream  {
     // Parse the input tokens into a syntax tree
     let input = parse_macro_input!(input as DeriveInput);
 
@@ -89,12 +90,26 @@ fn expand_enum(
     }
 
     let match_arms = e.variants.iter().try_fold(
-        TokenStream::new, |variant| {
-            let attrs = FmtAttribute::parse_attrs(&variant.attrs, attr_name)?
+        (Vec::new(), TokenStream::new()),
+        |mut arms, variant| {
+            let attrs = ContainerAttributes::parse_attrs(&variant.attrs, attr_name)?
                 .map(Spanning::into_inner)
                 .unwrap_or_default();
             let ident = &variant.ident;
 
+            if attrs.fmt.is_none()
+                && variant.fields.is_empty()
+                && attr_name != "display"
+            {
+                return Err(syn::Error::new(
+                    e.variants.span(),
+                    format!(
+                        "implicit formatting of unit enum variant is supported only for `Display` \
+                         macro, use `#[{attr_name}(\"...\")]` to explicitly specify the formatting",
+                    ),
+                ));
+            }
+
             let v = Expansion {
                 attrs: &attrs,
                 fields: &variant.fields,
@@ -121,7 +136,7 @@ fn expand_enum(
 
             Ok::<_, syn::Error>(arms)
         },
-    )?;
+    )?;    
 
     let body = match_arms
         .is_empty()

diff --git a/tokenizers/display_derive/src/parsing.rs b/tokenizers/display_derive/src/parsing.rs
@@ -194,9 +194,7 @@ pub(crate) fn format_string(input: &str) -> Option<FormatString<'_>> {
     let formats = iter::repeat(())
         .scan(&mut input, |input, _| {
             let (curr, format) =
-                alt(&mut [&mut maybe_format, &mut map(text, |(i, _)| (i, None))])(
-                    input,
-                )?;
+                alt(&mut [&mut maybe_format, &mut map(text, |(i, _)| (i, None))])(input)?;
             **input = curr;
             Some(format)
         })
@@ -600,9 +598,7 @@ fn lookahead(
 fn optional_result<'i, T: 'i>(
     mut parser: impl FnMut(&'i str) -> Option<(&'i str, T)>,
 ) -> impl FnMut(&'i str) -> (LeftToParse<'i>, Option<T>) {
-    move |input: &str| {
-        map_or_else(&mut parser, |i| (i, None), |(i, c)| (i, Some(c)))(input)
-    }
+    move |input: &str| map_or_else(&mut parser, |i| (i, None), |(i, c)| (i, Some(c)))(input)
 }
 
 /// Parses while `parser` is successful. Never fails.
@@ -682,9 +678,7 @@ fn char(c: char) -> impl FnMut(&str) -> Option<LeftToParse<'_>> {
 /// Checks whether first [`char`] suits `check`.
 ///
 /// [`char`]: fn@char
-fn check_char(
-    mut check: impl FnMut(char) -> bool,
-) -> impl FnMut(&str) -> Option<LeftToParse<'_>> {
+fn check_char(mut check: impl FnMut(char) -> bool) -> impl FnMut(&str) -> Option<LeftToParse<'_>> {
     move |input| {
         input
             .chars()
@@ -1159,9 +1153,7 @@ mod tests {
                         alternate: None,
                         zero_padding: None,
                         width: None,
-                        precision: Some(Precision::Count(Count::Parameter(
-                            Argument::Integer(0),
-                        ))),
+                        precision: Some(Precision::Count(Count::Parameter(Argument::Integer(0),))),
                         ty: Type::Display,
                     }),
                 }],
@@ -1178,9 +1170,9 @@ mod tests {
                         alternate: None,
                         zero_padding: None,
                         width: None,
-                        precision: Some(Precision::Count(Count::Parameter(
-                            Argument::Identifier("par"),
-                        ))),
+                        precision: Some(Precision::Count(Count::Parameter(Argument::Identifier(
+                            "par"
+                        ),))),
                         ty: Type::Display,
                     }),
                 }],
@@ -1197,9 +1189,9 @@ mod tests {
                         alternate: Some(Alternate),
                         zero_padding: None,
                         width: Some(Count::Parameter(Argument::Integer(2))),
-                        precision: Some(Precision::Count(Count::Parameter(
-                            Argument::Identifier("par"),
-                        ))),
+                        precision: Some(Precision::Count(Count::Parameter(Argument::Identifier(
+                            "par"
+                        ),))),
                         ty: Type::Display,
                     }),
                 }],
@@ -1250,9 +1242,9 @@ mod tests {
                         alternate: Some(Alternate),
                         zero_padding: None,
                         width: Some(Count::Parameter(Argument::Identifier("par"))),
-                        precision: Some(Precision::Count(Count::Parameter(
-                            Argument::Identifier("par"),
-                        ))),
+                        precision: Some(Precision::Count(Count::Parameter(Argument::Identifier(
+                            "par"
+                        ),))),
                         ty: Type::UpperDebug,
                     }),
                 }],

diff --git a/tokenizers/display_derive/src/vendored.rs b/tokenizers/display_derive/src/vendored.rs
@@ -1,14 +1,13 @@
-use syn::LitStr;
-use proc_macro2::TokenStream;
 use crate::parsing;
+use proc_macro2::TokenStream;
 use quote::{format_ident, ToTokens};
 use syn::{
     parse::{Parse, ParseStream},
     punctuated::Punctuated,
-    spanned::Spanned as _,
-    token,
+    token, 
     Expr,
 };
+
 /// Representation of a [`fmt`]-like attribute.
 ///
 /// ```rust,ignore
@@ -17,7 +16,7 @@ use syn::{
 ///
 /// [`fmt`]: std::fmt
 #[derive(Debug)]
-struct FmtAttribute {
+pub struct FmtAttribute {
     /// Interpolation [`syn::LitStr`].
     ///
     /// [`syn::LitStr`]: struct@syn::LitStr
@@ -42,7 +41,7 @@ impl Parse for FmtAttribute {
                 .peek(token::Comma)
                 .then(|| input.parse())
                 .transpose()?,
-            args: input.parse_terminated(FmtArgument::parse, token::Comma)?,
+            args: input.parse_terminated(FmtArgument::parse)?,
         })
     }
 }
@@ -70,8 +69,7 @@ impl FmtAttribute {
 
         // (1) There is exactly one formatting parameter.
         let lit = self.lit.value();
-        let param =
-            parsing::format(&lit).and_then(|(more, p)| more.is_empty().then_some(p))?;
+        let param = parsing::format(&lit).and_then(|(more, p)| more.is_empty().then_some(p))?;
 
         // (2) And the formatting parameter doesn't contain any modifiers.
         if param
@@ -164,4 +162,3 @@ impl ToTokens for FmtArgument {
         self.expr.to_tokens(tokens);
     }
 }
-