Updated transformers to 4.45.2

sillsdev · Oct 23, 2024 · 3da797f · 3da797f
1 parent 1415d12
commit 3da797f
Show file tree

Hide file tree

Showing 3 changed files with 225 additions and 207 deletions.
diff --git a/machine/translation/huggingface/hugging_face_nmt_model_trainer.py b/machine/translation/huggingface/hugging_face_nmt_model_trainer.py
@@ -227,7 +227,7 @@ def add_lang_code_to_tokenizer(tokenizer: Any, lang_code: str):
             lang_id = tokenizer.convert_tokens_to_ids(lang_code)
             tokenizer.lang_code_to_id[lang_code] = lang_id
 
-            if isinstance(tokenizer, (NllbTokenizer, MBart50Tokenizer, MBartTokenizer)):
+            if isinstance(tokenizer, (MBart50Tokenizer, MBartTokenizer)):
                 tokenizer.id_to_lang_code[lang_id] = lang_code
                 tokenizer.fairseq_tokens_to_ids[lang_code] = lang_id
                 tokenizer.fairseq_ids_to_tokens[lang_id] = lang_code
@@ -276,7 +276,7 @@ def add_lang_code_to_tokenizer(tokenizer: Any, lang_code: str):
 
             # For multilingual translation models like mBART-50 and M2M100 we need to force the target language token
             # as the first generated token. We ask the user to explicitly provide this as --forced_bos_token argument.
-            forced_bos_token_id = tokenizer.lang_code_to_id[self._tgt_lang]
+            forced_bos_token_id = tokenizer.convert_tokens_to_ids(self._tgt_lang)
             model.config.forced_bos_token_id = forced_bos_token_id
             if model.generation_config is not None:
                 model.generation_config.forced_bos_token_id = forced_bos_token_id