diff --git a/tokenizers/src/pre_tokenizers/byte_level.rs b/tokenizers/src/pre_tokenizers/byte_level.rs index 02366b280..2d3845b55 100644 --- a/tokenizers/src/pre_tokenizers/byte_level.rs +++ b/tokenizers/src/pre_tokenizers/byte_level.rs @@ -11,7 +11,7 @@ use crate::utils::macro_rules_attribute; /// Converts bytes to unicode characters. /// See https://github.com/openai/gpt-2/blob/master/src/encoder.py#L9 -pub fn bytes_char() -> HashMap { +pub(crate) fn bytes_char() -> HashMap { let mut bs: Vec = vec![]; bs.extend(b'!'..=b'~'); bs.extend(b'\xA1'..=b'\xAC');