update python test

huggingface · Jan 19, 2024 · af40ae2 · af40ae2
1 parent bf1dbe6
commit af40ae2
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/bindings/python/tests/bindings/test_tokenizer.py b/bindings/python/tests/bindings/test_tokenizer.py
@@ -465,7 +465,9 @@ def test_encode_special_tokens(self):
         output = tokenizer.encode("Hey there<end_of_text> dear<eot>friend!", add_special_tokens=False)
         assert output.tokens == ["▁Hey", "▁there", "<end_of_text>", "▁dear", "<eot>", "▁friend", "!"]
 
-        tokenizer.set_encode_special_tokens(True)
+        tokenizer.encode_special_tokens = True
+        assert tokenizer.encode_special_tokens == True
+
         output = tokenizer.encode("Hey there<end_of_text> dear<eot>friend!", add_special_tokens=False)
         assert output.tokens == [
             "▁Hey",