Add unit test for llama2 w/o chat template set

huggingface · Dec 16, 2023 · ec4d0c6 · ec4d0c6
1 parent d9b63ae
commit ec4d0c6
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/tests/generate_tests.py b/tests/generate_tests.py
@@ -18,6 +18,7 @@
     "llama": [
         'hf-internal-testing/llama-tokenizer',  # Special tokens: normalized=true
         'Xenova/llama2-tokenizer',  # Special tokens: normalized=false
+        'Xenova/llama2-chat-tokenizer', # Special tokens: normalized=false
         'hf-internal-testing/llama-code-tokenizer',
     ],
     'mpt': [
@@ -163,6 +164,11 @@
         'system',
         'system + assistant',
     ],
+    'Xenova/llama2-chat-tokenizer': [
+        'basic',
+        'system',
+        'system + assistant',
+    ],
 
     # TODO: add llama (v1) tokenizer once bug in transformers is fixed
     # https://github.com/huggingface/transformers/pull/26678