maybe work

huggingface · Jun 12, 2024 · 99cb054 · 99cb054
1 parent 292475f
commit 99cb054
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/tokenizers/display_derive/Cargo.toml b/tokenizers/display_derive/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 syn = "1.0"
 quote = "1.0"
 proc-macro2 = "1.0"
+unicode-xid = "0.2.4"
 
 [lib]
 proc-macro = true
diff --git a/tokenizers/display_derive/src/parsing.rs b/tokenizers/display_derive/src/parsing.rs
@@ -4,7 +4,7 @@
 
 use std::iter;
 
-// use unicode_xid::UnicodeXID as XID;
+use unicode_xid::UnicodeXID as XID;
 
 /// Output of the [`format_string`] parser.
 #[derive(Clone, Debug, Eq, PartialEq)]