Add unit test for Whitespace pretokenizer

huggingface · Jan 26, 2024 · 35c6415 · 35c6415
1 parent dbafde4
commit 35c6415
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/tests/generate_tests.py b/tests/generate_tests.py
@@ -28,6 +28,10 @@
         # TODO: Add back when https://github.com/huggingface/transformers/issues/26318 is fixed
         # 'Xenova/t5-tokenizer-new',
     ],
+    'bert': [
+        # Uses `Whitespace` pretokenizer 
+        'Xenova/jina-embeddings-v2-base-zh-tokenizer',
+    ],
 }
 
 MODELS_TO_IGNORE = [