diff --git a/tests/tokenizers.test.js b/tests/tokenizers.test.js index ff6e252f7..c055f5c06 100644 --- a/tests/tokenizers.test.js +++ b/tests/tokenizers.test.js @@ -77,6 +77,9 @@ describe('Tokenizers (hard-coded)', () => { ], } + // Re-use the same tests for the llama2 tokenizer + TESTS['Xenova/llama2-tokenizer'] = TESTS['Xenova/llama-tokenizer_new']; + for (const [tokenizerName, test_data] of Object.entries(TESTS)) { it(tokenizerName, async () => {