Make custom_detok settings backward compatible

contrib/benchmark.py

@@ -72,7 +72,7 @@ def benchmark(name, config, args):
 				"modifiers": [
 					{
 						"Tags": 0.1,
-						"custom_detok_trg": "moses:zh"
+						"custom_detok_trg": "zh"
 					}
 				]
 			}
@@ -85,7 +85,7 @@ def benchmark(name, config, args):
 				"modifiers": [
 					{
 						"Tags": 0.1,
-						"custom_detok_trg": "moses:zh",
+						"custom_detok_trg": "zh",
 						"spm_vocab": os.path.join(root, "test-data/vocab.zhen.spm")
 					}
 				]

contrib/test_enzh_config.yml

@@ -14,8 +14,8 @@ modifiers:
   - TitleCase: 0.2
   - Typos: 0.2
   - Tags: 0.1
-    custom_detok_src: "moses:null" # Null value for the src detokenizer
-    custom_detok_trg: "moses:zh"
+    custom_detok_src: null # Null value for the src detokenizer
+    custom_detok_trg: zh
     template: "__source__ {src} __target__ {trg} __done__" # This is the default way of inserting tags.
     # We STRONGLY DISCOURAGE the modification of this line and in fact it shouldn't be included in the config
     # unless you really know what you are doing.

contrib/test_enzh_noise_config.yml

@@ -13,8 +13,8 @@ modifiers:
   - UpperCase: 0.2
   - TitleCase: 0.2
   - Tags: 0.1
-    custom_detok_src: "moses:null" # Null value for the src detokenizer
-    custom_detok_trg: "moses:zh"
+    custom_detok_src: null # Null value for the src detokenizer
+    custom_detok_trg: zh
     augment: 0.4 # 0.4 out of the time tags is triggered, instead augment the source and the target with random noise
     replace: 0.4 # =====//=====, instead replace the target token with random noise, and use that random noise to tag a corresponding source word
     # template: "__source__ {src} __target__ {trg} __done__" # This is the default way of inserting tags.

contrib/test_enzh_tags_advanced_config.yml

@@ -15,8 +15,8 @@ start:
     - until clean 1
   modifiers:
     - Tags: 0.5
-      custom_detok_src: "moses:null"
-      custom_detok_trg: "moses:null"
+      custom_detok_src: null
+      custom_detok_trg: null
       template: "{src} __target__ {trg} __done__"
     - *modifiers
 
@@ -26,8 +26,8 @@ end:
     - until clean 1
   modifiers:
     - Tags: 0.5
-      custom_detok_src: "moses:null"
-      custom_detok_trg: "moses:zh"
+      custom_detok_src: null
+      custom_detok_trg: zh
       template: "{src} __target__ {trg} __done__"
     - *modifiers
 

contrib/test_enzh_tags_stage_config.yml

@@ -12,8 +12,8 @@ start:
     - until clean 1
   modifiers:
     - Tags: 0.5
-      custom_detok_src: "moses:null"
-      custom_detok_trg: "moses:null"
+      custom_detok_src: null
+      custom_detok_trg: null
       template: "{src} __target__ {trg} __done__"
 
 end:
@@ -22,8 +22,8 @@ end:
     - until clean 1
   modifiers:
     - Tags: 0.5
-      custom_detok_src: "moses:null"
-      custom_detok_trg: "moses:zh"
+      custom_detok_src: null
+      custom_detok_trg: zh
       template: "{src} __target__ {trg} __done__"
 
 seed: 1111

contrib/test_full_config.yml

@@ -52,8 +52,8 @@ modifiers:
     repeated_char: 0.1 # Repeats a random word character.
     unichar:       0.1 # Replaces a random consecutive repeated letter with a single letter.
   - Tags: 0.08
-    custom_detok_src: "moses:null"
-    custom_detok_trg: "moses:zh"
+    custom_detok_src: null
+    custom_detok_trg: zh
     template: "__source__ {src} __target__ {trg} __done__"
 
 seed: 1111

contrib/test_zhen_config.yml

@@ -12,7 +12,7 @@ start:
 modifiers:
   # No UpperCase or TitleCase modifier when the source is Chinese as we can't upper or lowercase Chinese
   - Tags: 0.1
-    custom_detok_src: "moses:zh"
+    custom_detok_src: zh
 
 seed: 1111
 trainer: cat

contrib/train_config.yml

@@ -31,8 +31,8 @@ modifiers:
   - UpperCase: 0.05
   - TitleCase: 0.05
   #- Tags: 0.08 # Requires dataset augmented with alignment info
-  #  custom_detok_src: "moses:null" # Null value for the src detokenizer
-  #  custom_detok_trg: "moses:zh"
+  #  custom_detok_src: null # Null value for the src detokenizer
+  #  custom_detok_trg: zh
   # template: "__source__ {src} __target__ {trg} __done__" # This is the default way of inserting tags.
     # We STRONGLY DISCOURAGE the modification of this line and in fact it shouldn't be included in the config
     # unless you really know what you are doing.

src/opustrainer/modifiers/placeholders.py

@@ -257,6 +257,13 @@ def __init__(self, probability: float=0.0, custom_detok_src: Optional[str]=None,
         super().__init__(probability)
 
         self.template = template
+
+        # uses Moses detokenizer by default
+        if custom_detok_src and ':' not in custom_detok_src:
+            custom_detok_src = f'moses:{custom_detok_src}'
+        if custom_detok_trg and ':' not in custom_detok_trg:
+            custom_detok_trg = f'moses:{custom_detok_trg}'
+
         self.custom_detok_src = custom_detok_src
         self.custom_detok_trg = custom_detok_trg
 

src/opustrainer/trainer.py

@@ -504,8 +504,8 @@ def _load_modifiers(self, ymldata:dict, basepath:str) -> List[Modifier]:
           - TitleCase: 0.05
           - Tags: 0.02
             num_tags: 6
-            custom_detok_src: "moses:null"
-            custom_detok_trg: "moses:zh"
+            custom_detok_src: null
+            custom_detok_trg: zh
         ```
         """
         modifiers = [

tests/test_placeholders.py

@@ -54,8 +54,8 @@ def test_retokenize(self):
     retokenize the input, and update the alignments accordingly."""
     tagger = PlaceholderTagModifier(
       probability=0.25,
-      custom_detok_src='moses:en',
-      custom_detok_trg='moses:zh',
+      custom_detok_src='en',
+      custom_detok_trg='zh',
       spm_vocab='contrib/test-data/vocab.zhen.spm') # type: ignore Path vs String type issue
 
     output = tagger(['\t'.join([
@@ -124,8 +124,8 @@ def test_retokenize_on_non_trigger(self):
     retokenize the input, even if probability is 0."""
     tagger = PlaceholderTagModifier(
       probability=0.0,
-      custom_detok_src='moses:en',
-      custom_detok_trg='moses:zh',
+      custom_detok_src='en',
+      custom_detok_trg='zh',
       spm_vocab='contrib/test-data/vocab.zhen.spm') # type: ignore Path vs String type issue
 
     output = tagger(['\t'.join([
@@ -148,7 +148,7 @@ def test_mode(self):
     multiple modes are enabled."""
     tagger = PlaceholderTagModifier(
       probability=1.0,
-      custom_detok_src='moses:zh',
+      custom_detok_src='zh',
       augment=0.33,
       replace=0.33,
       # tag=0.33 is implicit