bugfix: IL-414 retry dataset creation on error

src/intelligence_layer/evaluation/dataset/file_dataset_repository.py

@@ -23,9 +23,16 @@ def __init__(self, filesystem: AbstractFileSystem, root_directory: Path) -> None
         super().__init__(file_system=filesystem, root_directory=root_directory)
 
     def create_dataset(
-        self, examples: Iterable[Example[Input, ExpectedOutput]], dataset_name: str
+        self,
+        examples: Iterable[Example[Input, ExpectedOutput]],
+        dataset_name: str,
+        id: str | None = None,
     ) -> Dataset:
-        dataset = Dataset(name=dataset_name)
+        if id is None:
+            dataset = Dataset(name=dataset_name)
+        else:
+            dataset = Dataset(name=dataset_name, id=id)
+
         self.mkdir(self._dataset_directory(dataset.id))
 
         dataset_path = self._dataset_path(dataset.id)

src/intelligence_layer/evaluation/dataset/hugging_face_dataset_repository.py

@@ -1,10 +1,18 @@
+import time
 from pathlib import Path
-from typing import Optional
+from typing import Iterable, Optional
+import typing
 
 import huggingface_hub  # type: ignore
 from huggingface_hub import HfFileSystem, create_repo
+from huggingface_hub.utils import HfHubHTTPError
 
-from intelligence_layer.evaluation.dataset.domain import Dataset
+from intelligence_layer.core.task import Input
+from intelligence_layer.evaluation.dataset.domain import (
+    Dataset,
+    Example,
+    ExpectedOutput,
+)
 from intelligence_layer.evaluation.dataset.file_dataset_repository import (
     FileSystemDatasetRepository,
 )
@@ -42,6 +50,25 @@ def __init__(self, repository_id: str, token: str, private: bool) -> None:
         self._repository_id = repository_id
         self._file_system = file_system  # for better type checks
 
+    def create_dataset(
+        self,
+        examples: Iterable[Example[Input, ExpectedOutput]],
+        dataset_name: str,
+        id: str | None = None,
+    ) -> Dataset:
+        failures = 0
+        exception = None
+        while failures < 5:
+            try:
+                dataset = super().create_dataset(examples, dataset_name, id)
+                return dataset
+            except Exception as e:
+                exception = typing.cast(HfHubHTTPError, e)
+            failures += 1
+            print(f"Failure {failures}")
+            time.sleep(0.5)
+        raise exception  # RuntimeError("Cannot create dataset on Huggingface.")
+
     def delete_repository(self) -> None:
         huggingface_hub.delete_repo(
             repo_id=self._repository_id,

tests/evaluation/test_hugging_face_dataset_repository.py

@@ -50,14 +50,19 @@ def hugging_face_repository_with_dataset_and_examples(
     Tuple[HuggingFaceDatasetRepository, Dataset, Sequence[Example[str, str]]]
 ]:
     examples = [example_1, example_2]
-    dataset = hugging_face_dataset_repository.create_dataset(
-        examples=examples, dataset_name="test-hg-dataset"
-    )
+    id = str(uuid4())
+    try:
+        dataset = hugging_face_dataset_repository.create_dataset(
+            examples=examples, dataset_name="test-hg-dataset", id=id
+        )
+    except Exception as e:
+        hugging_face_dataset_repository.delete_dataset(id)
+        raise e
 
     try:
         yield hugging_face_dataset_repository, dataset, examples
     finally:
-        hugging_face_dataset_repository.delete_dataset(dataset.id)
+        hugging_face_dataset_repository.delete_dataset(id)
 
 
 def test_hugging_face_repository_can_create_and_delete_a_repository(