From 2d861a291b82729e1aa927935e464cdba761b513 Mon Sep 17 00:00:00 2001
From: Oskar <o.d.vanderwal@gmail.com>
Date: Thu, 28 Apr 2022 21:10:10 +0200
Subject: [PATCH] Added CrowsPairs for English and French

---
 lm_eval/tasks/__init__.py                 |  5 ++
 lm_eval/tasks/crows_pairs_multilingual.py | 65 +++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 lm_eval/tasks/crows_pairs_multilingual.py

diff --git a/lm_eval/tasks/__init__.py b/lm_eval/tasks/__init__.py
index ecd057891d..4191225d23 100644
--- a/lm_eval/tasks/__init__.py
+++ b/lm_eval/tasks/__init__.py
@@ -59,6 +59,7 @@
 from . import wino_bias
 from . import e2e_nlg_cleaned
 from . import gem_asset_turk
+from . import crows_pairs_multilingual
 
 ########################################
 # Translation tasks
@@ -317,6 +318,10 @@
     "wino_bias_type1_anti": wino_bias.WinoBiasType1Anti,
     "wino_bias_type2_pro": wino_bias.WinoBiasType2Pro,
     "wino_bias_type2_anti": wino_bias.WinoBiasType2Anti,
+
+    # Crows-Pairs
+    "crows_pairs_english": crows_pairs_multilingual.CrowsPairsEnglish,
+    "crows_pairs_french": crows_pairs_multilingual.CrowsPairsFrench,
 }
 
 
diff --git a/lm_eval/tasks/crows_pairs_multilingual.py b/lm_eval/tasks/crows_pairs_multilingual.py
new file mode 100644
index 0000000000..12d809f77f
--- /dev/null
+++ b/lm_eval/tasks/crows_pairs_multilingual.py
@@ -0,0 +1,65 @@
+"""
+French CrowS-Pairs: Extending a challenge dataset for measuring social bias in masked language models to a language other than English
+https://hal.inria.fr/hal-03629677/file/ACLFinal.pdf
+Measuring social biases in masked language models in English and French.
+https://gitlab.inria.fr/french-crows-pairs/acl-2022-paper-data-and-code/-/tree/main
+"""
+from lm_eval.base import PromptSourceTask
+
+
+_CITATION = """\
+@inproceedings{neveol2022french,
+  title={French CrowS-Pairs: Extending a challenge dataset for measuring social bias in masked language models to a language other than English},
+  author={N{\'e}v{\'e}ol, Aur{\'e}lie and Dupont, Yoann and Bezan{\c{c}}on, Julien and Fort, Kar{\"e}n},
+  booktitle={ACL 2022-60th Annual Meeting of the Association for Computational Linguistics},
+  year={2022}
+"""
+
+
+class CrowsPairsEnglish(PromptSourceTask):
+    VERSION = 0
+    DATASET_PATH = "oskarvanderwal/crows_pairs_multilingual"
+    DATASET_NAME = "english"
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return False
+
+    def has_test_docs(self):
+        return True
+
+    def training_docs(self):
+        pass
+      
+    def validation_docs(self):
+        pass
+    
+    def test_docs(self):
+        if self.has_test_docs():
+            return self.dataset["test"]
+          
+class CrowsPairsFrench(PromptSourceTask):
+    VERSION = 0
+    DATASET_PATH = "oskarvanderwal/crows_pairs_multilingual"
+    DATASET_NAME = "french"
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return False
+
+    def has_test_docs(self):
+        return True
+
+    def training_docs(self):
+        pass
+      
+    def validation_docs(self):
+        pass
+    
+    def test_docs(self):
+        if self.has_test_docs():
+            return self.dataset["test"]