diff --git a/src/utils/tokenize.rs b/src/utils/tokenize.rs index 13e5b43..6e7f445 100644 --- a/src/utils/tokenize.rs +++ b/src/utils/tokenize.rs @@ -64,10 +64,18 @@ impl< SAMRef: Deref>, > GreedyTokenizer { + pub fn get_sam(&self) -> &SAMRef { + &self.sam + } + pub fn get_sam_ref(&self) -> &GeneralSAM { &self.sam } + pub fn get_suffix_data(&self) -> &Vec> { + &self.suffix_data + } + pub fn inner_as_ref( &self, ) -> GreedyTokenizer> {