feature: Add retry logic in LimitedConcurrencyClient in case of `Bu…

…syError`

TASK: IL-431

src/intelligence_layer/connectors/limited_concurrency_client.py

@@ -1,6 +1,7 @@
 from functools import lru_cache
 from os import getenv
 from threading import Semaphore
+from time import sleep
 from typing import Any, Mapping, Optional, Protocol, Sequence
 
 from aleph_alpha_client import (
@@ -106,10 +107,12 @@ class LimitedConcurrencyClient:
     """
 
     def __init__(
-        self, client: AlephAlphaClientProtocol, max_concurrency: int = 20
+        self, client: AlephAlphaClientProtocol, max_concurrency: int = 20, number_of_retries: int = 10, seconds_between_retries: float = 1
     ) -> None:
         self._client = client
         self._concurrency_limit_semaphore = Semaphore(max_concurrency)
+        self._number_of_retries = number_of_retries
+        self._seconds_between_retries = seconds_between_retries
 
     @classmethod
     @lru_cache(maxsize=1)
@@ -143,7 +146,16 @@ def complete(
         model: str,
     ) -> CompletionResponse:
         with self._concurrency_limit_semaphore:
-            return self._client.complete(request, model)
+            for _ in range(self._number_of_retries):
+                try:
+                    return self._client.complete(request, model)
+                except Exception as e:
+                    if e.args[0] == 503:
+                        sleep(self._seconds_between_retries)
+                        continue
+                    else:
+                        raise e
+            raise e
 
     def get_version(self) -> str:
         with self._concurrency_limit_semaphore: