update support scripts

Scripts/cleanseTerms.py

@@ -0,0 +1,30 @@
+import os
+import re
+
+def unpack_record(record):
+    term = ""
+    text = ""
+    try:
+        (term, text) = record.split("=", 1)
+        text = text.strip()
+    except:
+        term = record
+
+    return term, text if text != "" else "EMPTY"
+
+def upper_repl(match):
+     return match.group(1) + "=" + match.group(2).upper() + match.group(3)
+
+for root, dirs, files in os.walk('.'):
+    for file in files:
+        if file.endswith('.txt'):
+            filename = os.path.join(root, file)
+            print(f"sorting {filename}")
+            with open(filename, "rt", encoding="utf-8") as f:
+                data = f.readlines()
+                data[0] = data[0].replace('', '')
+                for idx, record in enumerate(data):
+                    data[idx] = re.sub(r"(.+?)=(.)(.*)", upper_repl, data[idx])
+                data.sort()
+            with open(filename, "wt", encoding="utf-8") as f:
+                f.writelines(data)

Scripts/translateAutoSync.py

@@ -8,6 +8,9 @@
 
 import os
 import codecs
+from deep_translator import GoogleTranslator
+
+CHARS_MAX = 4500
 
 def unpack_record(record):
     term = ""
@@ -20,6 +23,16 @@ def unpack_record(record):
 
     return term, text if text != "" else "EMPTY"
 
+def translate_text(text, code):
+    text = text.replace("\\n", "{99}")
+    if len(text) > 3 and len(text) <= CHARS_MAX:
+        translated = GoogleTranslator(source="auto", target=code).translate(text) 
+    else:
+        translated = text
+    translated = translated.replace("{99}", "\\n")
+
+    return translated
+
 def readRecord(filename):
     # read file and split with "=" to dict
     dic = {}
@@ -48,7 +61,8 @@ def sync_file(input_file, output_file, code):
     # compare
     for key, value in inputDict.items():
         if key not in outputDict:
-            outputDict[key] = value
+            # outputDict[key] = value
+            outputDict[key] = translate_text(value, code)
             print(f"\t+ {output_file} add：{key}={value}")
     # write
     with open(output_file, "wt", encoding="utf-8") as f: