added translated prompts

OpenGPTX · Feb 20, 2024 · 7f5ac40 · 7f5ac40
1 parent 26b7be9
commit 7f5ac40
Show file tree

Hide file tree

Showing 5 changed files with 140 additions and 16 deletions.
diff --git a/lm_eval/tasks/opengptx/arcx.py b/lm_eval/tasks/opengptx/arcx.py
@@ -47,13 +47,34 @@
     "HU",
 ]
 
+PROMPT_WORDS = {
+    'BG': ('Въпрос', 'Отговор'),
+    'DA': ('Spørgsmål', 'Svar'),
+    'DE': ('Frage', 'Antwort'),
+    'ET': ('Küsimus', 'Vastus'),
+    'FI': ('Kysymys', 'Vastaa'),
+    'FR': ('Question', 'Réponse'),
+    'EL': ('Ερώτηση', 'Απάντηση'),
+    'IT': ('Domanda', 'Risposta'),
+    'LV': ('Jautājums', 'Atbilde'),
+    'LT': ('Klausimas', 'Atsakymas'),
+    'NL': ('Vraag', 'Antwoord'),
+    'PL': ('Pytanie', 'Odpowiedź'),
+    'PT-PT': ('Questão', 'Resposta'),
+    'RO': ('Întrebare', 'Răspuns'),
+    'SV': ('Fråga', 'Svar'),
+    'SK': ('Otázka', 'Odpoveď'),
+    'SL': ('Vprašanje', 'Odgovor'),
+    'ES': ('Pregunta', 'Respuesta'),
+    'CS': ('Otázka', 'Odpověď'),
+    'HU': ('Kérdés', 'Válasz')
+ }
+
 
 def construct_task(lang: str, split: str):
     class ARC(ARCBase):
-        def __init__(self, *args, **kwargs):
-            self.DATASET_NAME = f"{split}_{lang.upper()}"
-            super().__init__(*args, **kwargs)
-
+        QWORD, RWORD = PROMPT_WORDS.get(lang,("Question", "Answer"))
+        DATASET_NAME = f"{split}_{lang}"
     return ARC
 
 
@@ -68,7 +89,9 @@ def construct_all_tasks():
 class ARCBase(MultipleChoiceTask):
     VERSION = 0
     DATASET_PATH = "openGPT-X/arcx"
+    DATASET_NAME = None
     NUM_FEW_SHOT = 25
+    QWORD, RWORD = None, None
 
     def has_training_docs(self):
         return True
@@ -97,7 +120,7 @@ def _process_doc(self, doc):
         doc["answerKey"] = num_to_letter.get(doc["answerKey"], doc["answerKey"])
         out_doc = {
             "id": doc["id"],
-            "query": "Question: " + doc["question"] + "\nAnswer:",
+            "query": self.QWORD + ": " + doc["question"] + f"\n{self.RWORD}:",
             "choices": doc["choices"]["text"],
             "gold": ["A", "B", "C", "D", "E"].index(doc["answerKey"]),
         }

diff --git a/lm_eval/tasks/opengptx/gsm8kx.py b/lm_eval/tasks/opengptx/gsm8kx.py
@@ -55,6 +55,28 @@
     "HU",
 ]
 
+PROMPT_WORDS = {
+    'BG': ('Въпрос', 'Отговор'),
+    'DA': ('Spørgsmål', 'Svar'),
+    'DE': ('Frage', 'Antwort'),
+    'ET': ('Küsimus', 'Vastus'),
+    'FI': ('Kysymys', 'Vastaa'),
+    'FR': ('Question', 'Réponse'),
+    'EL': ('Ερώτηση', 'Απάντηση'),
+    'IT': ('Domanda', 'Risposta'),
+    'LV': ('Jautājums', 'Atbilde'),
+    'LT': ('Klausimas', 'Atsakymas'),
+    'NL': ('Vraag', 'Antwoord'),
+    'PL': ('Pytanie', 'Odpowiedź'),
+    'PT-PT': ('Questão', 'Resposta'),
+    'RO': ('Întrebare', 'Răspuns'),
+    'SV': ('Fråga', 'Svar'),
+    'SK': ('Otázka', 'Odpoveď'),
+    'SL': ('Vprašanje', 'Odgovor'),
+    'ES': ('Pregunta', 'Respuesta'),
+    'CS': ('Otázka', 'Odpověď'),
+    'HU': ('Kérdés', 'Válasz')
+ }
 
 def construct_all_tasks():
     return {f"gsm8kx_{lang.lower()}": construct_task(lang) for lang in LANGS}
@@ -63,6 +85,7 @@ def construct_all_tasks():
 def construct_task(lang):
     class task(GradeSchoolMath8K):
         DATASET_NAME = lang
+        QWORD, RWORD = PROMPT_WORDS.get(lang,("Question", "Answer"))
 
     return task
 
@@ -74,6 +97,7 @@ class task(GradeSchoolMath8K):
 class GradeSchoolMath8K(Task):
     VERSION = 0
     DATASET_PATH = "openGPT-X/gsm8kx"
+    QWORD, RWORD = None, None
 
     def has_training_docs(self):
         return True
@@ -94,7 +118,7 @@ def test_docs(self):
         return self.dataset["test"]
 
     def doc_to_text(self, doc):
-        return "Question: " + doc["question"] + "\nAnswer:"
+        return self.QWORD + ": " + doc["question"] + f"\n{self.RWORD}:"
 
     def doc_to_target(self, doc):
         return " " + doc["answer"]
@@ -112,7 +136,7 @@ def construct_requests(self, doc, ctx):
         """
         # NOTE: The paper implements "verifiers" that assign a score to multiple
         # solutions and output the highest ranked solution.
-        completion = rf.greedy_until(ctx, {"until": [":", "Question:", "Question"]})
+        completion = rf.greedy_until(ctx, {"until": [":", f"{self.QWORD}:", f"{self.QWORD}"]})
         return completion
 
     def _extract_answer(self, completion):

diff --git a/lm_eval/tasks/opengptx/hellaswagx.py b/lm_eval/tasks/opengptx/hellaswagx.py
@@ -26,7 +26,7 @@
 }
 """
 
-LANGS = ["DE", "FR", "IT", "ES"]
+LANGS = ["BG","DA","DE","ET","FI","FR","EL","IT","LV","LT","NL","PL","PT-PT","RO","SV","SK","SL","ES","CS","HU"]
 
 
 def construct_all_tasks():

diff --git a/lm_eval/tasks/opengptx/mmlux.py b/lm_eval/tasks/opengptx/mmlux.py
@@ -107,6 +107,28 @@
     "world_religions",
 ]
 
+PROMPT_WORDS = {
+    'BG': ('Въпрос', 'Избори', 'Отговор'),
+    'DA': ('Spørgsmål', 'Valgmuligheder', 'Svar'),
+    'DE': ('Frage', 'Auswahlmöglichkeiten', 'Antwort'),
+    'ET': ('Küsimus', 'Valikud', 'Vastus'),
+    'FI': ('Kysymys', 'Valinnat', 'Vastaa'),
+    'FR': ('Question', 'Choix', 'Réponse'),
+    'EL': ('Ερώτηση', 'Επιλογές', 'Απάντηση'),
+    'IT': ('Domanda', 'Scelte', 'Risposta'),
+    'LV': ('Jautājums', 'Izvēle', 'Atbilde'),
+    'LT': ('Klausimas', 'Pasirinkimai', 'Atsakymas'),
+    'NL': ('Vraag', 'Keuzes', 'Antwoord'),
+    'PL': ('Pytanie', 'Wybory', 'Odpowiedź'),
+    'PT-PT': ('Questão', 'Escolhas', 'Resposta'),
+    'RO': ('Întrebare', 'Alegeri', 'Răspuns'),
+    'SV': ('Fråga', 'Valmöjligheter', 'Svar'),
+    'SK': ('Otázka', 'Voľby', 'Odpoveď'),
+    'SL': ('Vprašanje', 'Izbira', 'Odgovor'),
+    'ES': ('Pregunta', 'Opciones', 'Respuesta'),
+    'CS': ('Otázka', 'Volby', 'Odpověď'),
+    'HU': ('Kérdés', 'Választások', 'Válasz')
+ }
 
 def create_all_tasks():
     """Creates a dictionary of tasks from a list of subjects
@@ -121,9 +143,10 @@ def create_all_tasks():
 
 
 def create_task(subject, lang):
+    words = PROMPT_WORDS.get(lang,("Question", "Choices", "Answer"))
     class HendrycksTest(GeneralHendrycksTest):
         def __init__(self):
-            super().__init__(subject, lang)
+            super().__init__(subject, lang, words)
 
     return HendrycksTest
 
@@ -133,8 +156,9 @@ class GeneralHendrycksTest(MultipleChoiceTask):
     DATASET_PATH = "openGPT-X/mmlux"
     DATASET_NAME = None
 
-    def __init__(self, subject, lang):
+    def __init__(self, subject, lang, words):
         self.DATASET_NAME = f"{subject}_{lang}"
+        self.QWORD, self.CWORD, self.RWORD = words
         super().__init__()
 
     def has_training_docs(self):
@@ -163,11 +187,11 @@ def format_example(doc, keys):
             D. <choice4>
             Answer:
             """
-            prompt = "Question: " + doc["question"] + "\nChoices:\n"
+            prompt = f"{self.QWORD}: " + doc["question"] + f"\n{self.CWORD}:\n"
             prompt += "".join(
                 [f"{key}. {choice}\n" for key, choice in zip(keys, doc["choices"])]
             )
-            prompt += "Answer:"
+            prompt += f"{self.RWORD}:"
             return prompt
 
         keys = ["A", "B", "C", "D"]
-Original file line number
+Diff line change
@@ Expand Up / @@ -26,7 +26,7 @@ @@
     }
     """
-    LANGS = ["DE", "FR", "IT", "ES"]
+    LANGS = ["BG","DA","DE","ET","FI","FR","EL","IT","LV","LT","NL","PL","PT-PT","RO","SV","SK","SL","ES","CS","HU"]
     def construct_all_tasks():
@@ Expand Down @@