Change default texts value for blank samples

raileymontalan · Feb 27, 2024 · 911a582 · 911a582
1 parent e9f57a0
commit 911a582
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 5 deletions.
diff --git a/seacrowd/sea_datasets/wit/__init__.py b/seacrowd/sea_datasets/wit/__init__.py
diff --git a/seacrowd/sea_datasets/wit/wit.py b/seacrowd/sea_datasets/wit/wit.py
@@ -91,8 +91,6 @@
 
 _SEACROWD_VERSION = "1.0.0"
 
-logger = datasets.logging.get_logger(__name__)
-
 
 class WITDataset(datasets.GeneratorBasedBuilder):
     """
@@ -205,7 +203,7 @@ def _split_generators(self, dl_manager: datasets.DownloadManager) -> List[datase
                 name=datasets.Split.VALIDATION,
                 gen_kwargs={
                     "filepaths": val_paths,
-                    "split": "val",
+                    "split": "validation",
                 },
             ),
         ]
@@ -238,12 +236,23 @@ def _generate_examples(self, filepaths: Path, split: str) -> Tuple[int, Dict]:
                 if self.config.schema == "seacrowd_imtext":
                     for d in data:
                         if d["language"] in language_list:
+                            text = None
+                            context = None
+                            if d["caption_reference_description"] != "":
+                                text = d["caption_reference_description"]
+                                context = "caption_reference_description"
+                            elif d["caption_attribution_description"] != "":
+                                text = d["caption_attribution_description"]
+                                context = "caption_attribution_description"
+                            else:
+                                text = d["caption_alt_text_description"]
+                                context = "caption_alt_text_description"
                             x = {
                                 "id": idx,
                                 "image_paths": [d["image_url"]],
-                                "texts": d["page_title"],
+                                "texts": text,
                                 "metadata": {
-                                    "context": None,
+                                    "context": context,
                                     "labels": None,
                                 },
                             }