adding missing character updates to machine.py

sillsdev · Oct 12, 2023 · eb2bbb6 · eb2bbb6
1 parent adf9bb9
commit eb2bbb6
Show file tree

Hide file tree

Showing 6 changed files with 772,436 additions and 2 deletions.
diff --git a/machine/jobs/huggingface/hugging_face_nmt_model_factory.py b/machine/jobs/huggingface/hugging_face_nmt_model_factory.py
@@ -59,6 +59,7 @@ def create_model_trainer(self, corpus: ParallelTextCorpus) -> Trainer:
             corpus,
             src_lang=self._config.src_lang,
             tgt_lang=self._config.trg_lang,
+            config=self._config,
         )
 
     def create_engine(self) -> TranslationEngine:

diff --git a/machine/jobs/settings.yaml b/machine/jobs/settings.yaml
@@ -3,6 +3,9 @@ default:
   max_steps: 20000
   data_dir: ~/machine
   batch_size: 1024
+  tokenizer:
+    update_src: false
+    update_trg: false
   huggingface:
     parent_model_name: facebook/nllb-200-1.3B
     train_params: