Update tokenizers/src/pre_tokenizers/byte_level.rs

Co-authored-by: Luc Georges <[email protected]>

tokenizers/src/pre_tokenizers/byte_level.rs

@@ -11,7 +11,7 @@ use crate::utils::macro_rules_attribute;
 
 /// Converts bytes to unicode characters.
 /// See https://github.com/openai/gpt-2/blob/master/src/encoder.py#L9
-pub fn bytes_char() -> HashMap<u8, char> {
+pub(crate) fn bytes_char() -> HashMap<u8, char> {
     let mut bs: Vec<u8> = vec![];
     bs.extend(b'!'..=b'~');
     bs.extend(b'\xA1'..=b'\xAC');