From fc16dc72980f90e30bcb46a4cae9c3c75f206ef0 Mon Sep 17 00:00:00 2001
From: Joshua Lochner <admin@xenova.com>
Date: Wed, 21 Feb 2024 15:34:14 +0200
Subject: [PATCH] Add Gemma tokenizer unit test

---
 tests/generate_tests.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/generate_tests.py b/tests/generate_tests.py
index 81836b901..40725f812 100644
--- a/tests/generate_tests.py
+++ b/tests/generate_tests.py
@@ -36,6 +36,9 @@
         # Uses a pretokenizer regex which is not compatible with JavaScript.
         'Qwen/Qwen1.5-0.5B-Chat',
     ],
+    'gemma': [
+        'hf-internal-testing/dummy-gemma',
+    ],
 }
 
 MODELS_TO_IGNORE = [