feat: export inner data of GreedyTokenizer (#42)

ModelTC · Nov 27, 2023 · 02eebe6 · 02eebe6
1 parent f59629b
commit 02eebe6
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/src/utils/tokenize.rs b/src/utils/tokenize.rs
@@ -64,10 +64,18 @@ impl<
         SAMRef: Deref<Target = GeneralSAM<TransTable>>,
     > GreedyTokenizer<TransTable, TokenIDType, SAMRef>
 {
+    pub fn get_sam(&self) -> &SAMRef {
+        &self.sam
+    }
+
     pub fn get_sam_ref(&self) -> &GeneralSAM<TransTable> {
         &self.sam
     }
 
+    pub fn get_suffix_data(&self) -> &Vec<SuffixInTrieData<TokenIDType>> {
+        &self.suffix_data
+    }
+
     pub fn inner_as_ref(
         &self,
     ) -> GreedyTokenizer<TransTable, TokenIDType, &GeneralSAM<TransTable>> {