From 10492f4b97975c70a3c6e89f04f87aebf5fa9121 Mon Sep 17 00:00:00 2001
From: jon-tow <jon-tow@Jons-MacBook-Pro.local>
Date: Thu, 28 Apr 2022 10:49:12 -0400
Subject: [PATCH] Add `axg` and `axb` to `super_glue`

---
 lm_eval/tasks/superglue.py | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/lm_eval/tasks/superglue.py b/lm_eval/tasks/superglue.py
index 667dc54271..e1ebe5d189 100644
--- a/lm_eval/tasks/superglue.py
+++ b/lm_eval/tasks/superglue.py
@@ -305,3 +305,39 @@ def higher_is_better(self):
 
     def aggregation(self):
         return {"acc": mean}
+
+
+class WinogenderSchemaDiagnostics(PromptSourceTask):
+    VERSION = 0
+    DATASET_PATH = "super_glue"
+    DATASET_NAME = "axg"
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return False 
+
+    def has_test_docs(self):
+        return True 
+
+    def test_docs(self):
+        return self.dataset["test"]
+
+
+class BroadcoverageDiagnostics(PromptSourceTask):
+    VERSION = 0
+    DATASET_PATH = "super_glue"
+    DATASET_NAME = "axb"
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return False 
+
+    def has_test_docs(self):
+        return True 
+
+    def test_docs(self):
+        return self.dataset["test"]