Add quick doc to byte_level.rs (#1420)

* Add quick doc to byte_level.rs * Address PR comments
huggingface · Jan 3, 2024 · f1c23b8 · f1c23b8
1 parent 1146259
commit f1c23b8
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/tokenizers/src/pre_tokenizers/byte_level.rs b/tokenizers/src/pre_tokenizers/byte_level.rs
@@ -9,6 +9,8 @@ use crate::tokenizer::{
 };
 use crate::utils::macro_rules_attribute;
 
+/// Converts bytes to unicode characters.
+/// See https://github.com/openai/gpt-2/blob/master/src/encoder.py#L9
 fn bytes_char() -> HashMap<u8, char> {
     let mut bs: Vec<u8> = vec![];
     bs.extend(b'!'..=b'~');
@@ -33,6 +35,8 @@ fn bytes_char() -> HashMap<u8, char> {
 }
 
 lazy_static! {
+    /// Regex that matches exactly one token.
+    /// See https://github.com/openai/gpt-2/blob/master/src/encoder.py#L98
     static ref RE: SysRegex = SysRegex::new(
         r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"
     )