diff --git a/machine/tokenization/__init__.py b/machine/tokenization/__init__.py index aab2dbf..5084b67 100644 --- a/machine/tokenization/__init__.py +++ b/machine/tokenization/__init__.py @@ -9,6 +9,7 @@ from .string_tokenizer import StringTokenizer from .tokenization_utils import get_ranges, split from .tokenizer import Tokenizer +from .tokenizer_factory import create_detokenizer, create_tokenizer from .whitespace_detokenizer import WHITESPACE_DETOKENIZER, WhitespaceDetokenizer from .whitespace_tokenizer import WHITESPACE_TOKENIZER, WhitespaceTokenizer from .zwsp_word_detokenizer import ZwspWordDetokenizer @@ -27,6 +28,8 @@ "StringDetokenizer", "StringTokenizer", "Tokenizer", + "create_detokenizer", + "create_tokenizer", "WHITESPACE_DETOKENIZER", "WHITESPACE_TOKENIZER", "WhitespaceDetokenizer",