Fix Test

(cherry picked from commit fd220db)
apaniukov · Apr 9, 2024 · 909f6e0 · 909f6e0
1 parent 3f93427
commit 909f6e0
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tests/tokenizers_test.py b/tests/tokenizers_test.py
@@ -498,7 +498,7 @@ def test_eos_token_id_rt_info_bpe(hf_bpe_tokenizers):
 
 
 def test_eos_token_id_rt_info_tiktoken(hf_tiktoken_tokenizers):
-    eos_token_id = hf_tiktoken_tokenizers.eos_token_id
+    eos_token_id = hf_tiktoken_tokenizers.eos_token_id or hf_tiktoken_tokenizers.eod_id
     ov_tokenizer, ov_detokenizer = convert_tokenizer(
         hf_tiktoken_tokenizers,
         with_detokenizer=True,