Adds retries to redis client, closes #1734.

cvisionai · Jun 20, 2024 · 36319a4 · 36319a4
1 parent b8e1b0e
commit 36319a4
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 3 deletions.
diff --git a/api/main/cache.py b/api/main/cache.py
@@ -1,4 +1,11 @@
-import redis
+from redis.backoff import ExponentialBackoff
+from redis.retry import Retry
+from redis.client import Redis
+from redis.exceptions import (
+    BusyLoadingError,
+    ConnectionError,
+    TimeoutError,
+)
 import json
 import os
 import logging
@@ -16,8 +23,11 @@ class TatorCache:
 
     @classmethod
     def setup_redis(cls):
-        cls.rds = redis.Redis(
+        retry = Retry(ExponentialBackoff(), 3)
+        cls.rds = Redis(
             host=os.getenv("REDIS_HOST"),
+            retry=retry,
+            retry_on_error=[BusyLoadingError, ConnectionError, TimeoutError],
             health_check_interval=30,
             ssl=REDIS_USE_SSL,
         )

diff --git a/api/main/worker.py b/api/main/worker.py
@@ -1,4 +1,11 @@
-from redis import Redis
+from redis.backoff import ExponentialBackoff
+from redis.retry import Retry
+from redis.client import Redis
+from redis.exceptions import (
+    BusyLoadingError,
+    ConnectionError,
+    TimeoutError,
+)
 from rq import Queue, Worker
 
 import os
@@ -28,8 +35,11 @@ def push_job(queue, function, *args, **kwargs):
     push_job('async_jobs', print, 'Hello')
     See [https://python-rq.org/docs/] for information on available kwargs
     """
+    retry = Retry(ExponentialBackoff(), 3)
     redis = Redis(
         host=os.getenv("REDIS_HOST"),
+        retry=retry,
+        retry_on_error=[BusyLoadingError, ConnectionError, TimeoutError],
         ssl=REDIS_USE_SSL,
     )
     queue = Queue(queue, connection=redis)
@@ -41,8 +51,11 @@ def push_job(queue, function, *args, **kwargs):
     parser = argparse.ArgumentParser(description="Processes a python-rq queue.")
     parser.add_argument("queue", help="Name of queue to process.")
     args = parser.parse_args()
+    retry = Retry(ExponentialBackoff(), 3)
     redis = Redis(
         host=os.getenv("REDIS_HOST"),
+        retry=retry,
+        retry_on_error=[BusyLoadingError, ConnectionError, TimeoutError],
         ssl=REDIS_USE_SSL,
     )
     queue = Queue(args.queue, connection=redis)