From 4cc46c7125fce23202e0495e947e04981c7a9e5f Mon Sep 17 00:00:00 2001
From: John Lambert <john_lambert@sil.org>
Date: Fri, 23 Aug 2024 15:30:25 -0400
Subject: [PATCH] one more fix

---
 machine/tokenization/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/machine/tokenization/__init__.py b/machine/tokenization/__init__.py
index aab2dbf3..5084b67c 100644
--- a/machine/tokenization/__init__.py
+++ b/machine/tokenization/__init__.py
@@ -9,6 +9,7 @@
 from .string_tokenizer import StringTokenizer
 from .tokenization_utils import get_ranges, split
 from .tokenizer import Tokenizer
+from .tokenizer_factory import create_detokenizer, create_tokenizer
 from .whitespace_detokenizer import WHITESPACE_DETOKENIZER, WhitespaceDetokenizer
 from .whitespace_tokenizer import WHITESPACE_TOKENIZER, WhitespaceTokenizer
 from .zwsp_word_detokenizer import ZwspWordDetokenizer
@@ -27,6 +28,8 @@
     "StringDetokenizer",
     "StringTokenizer",
     "Tokenizer",
+    "create_detokenizer",
+    "create_tokenizer",
     "WHITESPACE_DETOKENIZER",
     "WHITESPACE_TOKENIZER",
     "WhitespaceDetokenizer",