Add Gemma tokenizer unit test

huggingface · Feb 21, 2024 · fc16dc7 · fc16dc7
1 parent 2449f1c
commit fc16dc7
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/tests/generate_tests.py b/tests/generate_tests.py
@@ -36,6 +36,9 @@
         # Uses a pretokenizer regex which is not compatible with JavaScript.
         'Qwen/Qwen1.5-0.5B-Chat',
     ],
+    'gemma': [
+        'hf-internal-testing/dummy-gemma',
+    ],
 }
 
 MODELS_TO_IGNORE = [