py : fix missing added_tokens_dict for SPM vocab

ggerganov · Jan 16, 2024 · 9b464b4 · 9b464b4
1 parent a0b3ac8
commit 9b464b4
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/convert.py b/convert.py
@@ -466,6 +466,7 @@ def __init__(
             )
 
         # Token pieces that were added to the base vocabulary.
+        self.added_tokens_dict = added_tokens
         self.added_tokens_list = [new_tokens[id] for id in actual_new_ids]
         self.vocab_size_base = vocab_size
         self.vocab_size = self.vocab_size_base + len(self.added_tokens_list)