Del Unused Regex

apaniukov · Nov 11, 2024 · 7055c35 · 7055c35
1 parent 9554fe5
commit 7055c35
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/python/openvino_tokenizers/tokenizer_pipeline.py b/python/openvino_tokenizers/tokenizer_pipeline.py
@@ -231,13 +231,6 @@ def del_control_chars_regex(cls) -> "RegexNormalizationStep":
             replace_term="",
         )
 
-    @classmethod
-    def clean_up_tokenization_spaces(cls) -> "RegexNormalizationStep":
-        return cls(
-            regex_search_pattern=r"(?| ([\.\?\!\,])| ('[ms])| (') | ('[rv]e))",
-            replace_term="$1",
-        )
-
     def get_ov_subgraph(self, input_nodes: List[Output]) -> List[Output]:
         input_nodes.extend(
             (

diff --git a/tests/layer_tests.py b/tests/layer_tests.py
@@ -136,6 +136,14 @@ def test_charsmap_normalizartion(test_string, hf_charsmap_tokenizer, precompiled
         (" Hello world!", " Hello world!", RegexNormalizationStep.add_prefix_whitespace_to_not_whitespace_regex()),
         ("\tHello world!", " \tHello world!", RegexNormalizationStep.add_prefix_whitespace_to_not_whitespace_regex()),
         ("\tHello", "▁\tHello", RegexNormalizationStep.prepend_regex("▁")),
+        (  # test backward compatibility with old regex
+            " ' declare",
+            "'declare",
+            RegexNormalizationStep(
+                regex_search_pattern=r" ([\\.\\?\\!,])| ('[ms])| (') | ('[rv]e)| (n't)",
+                replace_term=r"\1",
+            )
+        ),
     ]
 )
 def test_regex_normalization(test_string, expected, layer):