Add qwen-specific tokenizer unit test

huggingface · Feb 5, 2024 · ebd3bcd · ebd3bcd
1 parent 8fdeb78
commit ebd3bcd
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/tests/generate_tests.py b/tests/generate_tests.py
@@ -29,9 +29,13 @@
         # 'Xenova/t5-tokenizer-new',
     ],
     'bert': [
-        # Uses `Whitespace` pretokenizer 
+        # Uses `Whitespace` pretokenizer
         'Xenova/jina-embeddings-v2-base-zh-tokenizer',
     ],
+    'qwen2': [
+        # Uses a pretokenizer regex which is not compatible with JavaScript.
+        'Qwen/Qwen1.5-0.5B-Chat',
+    ],
 }
 
 MODELS_TO_IGNORE = [
@@ -137,6 +141,10 @@
             # Special treatment of characters in certain language
             "ț ţ",
         ],
+
+        "qwen2": [
+            "i'm i'M i've i've i'Ve i'vE i'VE",
+        ],
     },
     "custom": {
         "facebook/blenderbot_small-90M": [