Raise exception when CreateFleet doesn't return any instance

Raise an exception when CreateFleet doesn't return any instance and the error list in the CreateFleet contains only one error entry.
The exception raised is built with the same error code coming from the CreateFleet response, so that this error code can be set into the reason when putting the Slurm nodes into DOWN.
This is useful to avoid to trigger the fast capacity failover (error code InsufficientInstanceCapacity) when the CreateFleet call doesn't return any instance because of throttling (error code RequestLimitExceeded).

Signed-off-by: Luca Carrogu <[email protected]>

src/slurm_plugin/fleet_manager.py

@@ -71,6 +71,14 @@ def __init__(self, message: str):
         super().__init__(message)
 
 
+class LaunchInstancesError(Exception):
+    """Represent an error during the launch of EC2 instances"""
+
+    def __init__(self, code: str, message: str = ""):
+        self.code = code
+        super().__init__(message)
+
+
 class FleetManagerFactory:
     @staticmethod
     def get_manager(
@@ -361,7 +369,8 @@ def _launch_instances(self, launch_params):
 
             instances = response.get("Instances", [])
             log_level = logging.WARNING if instances else logging.ERROR
-            for err in response.get("Errors", []):
+            err_list = response.get("Errors", [])
+            for err in err_list:
                 logger.log(
                     log_level,
                     "Error in CreateFleet request (%s): %s - %s",
@@ -375,6 +384,8 @@ def _launch_instances(self, launch_params):
             if partial_instance_ids:
                 logger.error("Unable to retrieve instance info for instances: %s", partial_instance_ids)
 
+            if not instances and len(err_list) == 1:
+                raise LaunchInstancesError(err_list[0].get("ErrorCode"), err_list[0].get("ErrorMessage"))
             return {"Instances": instances}
         except ClientError as e:
             logger.error("Failed CreateFleet request: %s", e.response.get("ResponseMetadata", {}).get("RequestId"))

src/slurm_plugin/instance_manager.py

@@ -1016,6 +1016,8 @@ def _launch_instances(
                             update_failed_nodes_parameters = {"nodeset": set(batch_nodes)}
                             if isinstance(e, ClientError):
                                 update_failed_nodes_parameters["error_code"] = e.response.get("Error", {}).get("Code")
+                            if isinstance(e, Exception) and hasattr(e, "code"):
+                                update_failed_nodes_parameters["error_code"] = e.code
                             self._update_failed_nodes(**update_failed_nodes_parameters)
 
                             if job and all_or_nothing_batch: