Add llama2 static tests

huggingface · Dec 18, 2023 · 2958984 · 2958984
1 parent 4ddd1c8
commit 2958984
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/tests/tokenizers.test.js b/tests/tokenizers.test.js
@@ -77,6 +77,9 @@ describe('Tokenizers (hard-coded)', () => {
         ],
     }
 
+    // Re-use the same tests for the llama2 tokenizer
+    TESTS['Xenova/llama2-tokenizer'] = TESTS['Xenova/llama-tokenizer_new'];
+
     for (const [tokenizerName, test_data] of Object.entries(TESTS)) {
 
         it(tokenizerName, async () => {