one more fix

sillsdev · Aug 23, 2024 · 4cc46c7 · 4cc46c7
1 parent 4f50254
commit 4cc46c7
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/machine/tokenization/__init__.py b/machine/tokenization/__init__.py
@@ -9,6 +9,7 @@
 from .string_tokenizer import StringTokenizer
 from .tokenization_utils import get_ranges, split
 from .tokenizer import Tokenizer
+from .tokenizer_factory import create_detokenizer, create_tokenizer
 from .whitespace_detokenizer import WHITESPACE_DETOKENIZER, WhitespaceDetokenizer
 from .whitespace_tokenizer import WHITESPACE_TOKENIZER, WhitespaceTokenizer
 from .zwsp_word_detokenizer import ZwspWordDetokenizer
@@ -27,6 +28,8 @@
     "StringDetokenizer",
     "StringTokenizer",
     "Tokenizer",
+    "create_detokenizer",
+    "create_tokenizer",
     "WHITESPACE_DETOKENIZER",
     "WHITESPACE_TOKENIZER",
     "WhitespaceDetokenizer",