diff --git a/tokenizers/display_derive/src/vendored.rs b/tokenizers/display_derive/src/vendored.rs index 231858ed0..eccc5263a 100644 --- a/tokenizers/display_derive/src/vendored.rs +++ b/tokenizers/display_derive/src/vendored.rs @@ -1,11 +1,12 @@ +use std::{env::VarError, error::Error}; + use crate::parsing; use proc_macro2::TokenStream; use quote::{format_ident, ToTokens}; use syn::{ parse::{Parse, ParseStream}, punctuated::Punctuated, - token, - Expr, + token, Expr, }; /// Representation of a [`fmt`]-like attribute. @@ -32,19 +33,19 @@ pub struct FmtAttribute { impl Parse for FmtAttribute { fn parse(input: ParseStream<'_>) -> syn::Result { - - Ok(Self { + let attribute = Self { lit: input.parse()?, comma: input .peek(token::Comma) .then(|| input.parse()) .transpose()?, args: input.parse_terminated(FmtArgument::parse)?, - }) + }; + println!("Parsing FMTAttribute, {}, ",attribute.lit.token().to_string()); + Ok(attribute) } } - impl ToTokens for FmtAttribute { fn to_tokens(&self, tokens: &mut TokenStream) { self.lit.to_tokens(tokens); diff --git a/tokenizers/src/decoders/byte_fallback.rs b/tokenizers/src/decoders/byte_fallback.rs index 9c02acf2a..8c88199de 100644 --- a/tokenizers/src/decoders/byte_fallback.rs +++ b/tokenizers/src/decoders/byte_fallback.rs @@ -1,6 +1,6 @@ use crate::tokenizer::{Decoder, Result}; -use monostate::MustBe; use display_derive::Display; +use monostate::MustBe; use serde::{Deserialize, Serialize}; #[derive(Deserialize, Clone, Debug, Serialize, Default, Display)] /// ByteFallback is a simple trick which converts tokens looking like `<0x61>`