From d103771a58d2bf64f1242097858eb935fc8424cd Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Tue, 20 Feb 2024 14:12:13 -0500
Subject: [PATCH 1/2] Also retry on requests.exceptions.HTTPError

Got fresh failure

    [INFO    ] bids <- docker bids/afni_proc
    Traceback (most recent call last):
      File "/home/yoh/proj/repronim/containers/scripts/create_singularities", line 499, in <module>
        main()
      File "/usr/lib/python3/dist-packages/click/core.py", line 1130, in __call__
        return self.main(*args, **kwargs)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/usr/lib/python3/dist-packages/click/core.py", line 1055, in main
        rv = self.invoke(ctx)
             ^^^^^^^^^^^^^^^^
      File "/usr/lib/python3/dist-packages/click/core.py", line 1404, in invoke
        return ctx.invoke(self.callback, **ctx.params)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/usr/lib/python3/dist-packages/click/core.py", line 760, in invoke
        return __callback(*args, **kwargs)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/home/yoh/proj/repronim/containers/scripts/create_singularities", line 414, in main
        builder.generate_singularity_for_docker_image(dockerhubid, "bids")
      File "/home/yoh/proj/repronim/containers/scripts/create_singularities", line 303, in generate_singularity_for_docker_image
        last_version = self.get_last_docker_version_tag(
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/home/yoh/proj/repronim/containers/scripts/create_singularities", line 83, in get_last_docker_version_tag
        r.raise_for_status()
      File "/usr/lib/python3/dist-packages/requests/models.py", line 1021, in raise_for_status
        raise HTTPError(http_error_msg, response=self)
    requests.exceptions.HTTPError: 502 Server Error: Bad Gateway for url: https://registry.hub.docker.com/v2/repositories/bids/afni_proc/tags

and apparently they aren't of the same "hierarchy"

    In [5]: issubclass(requests.exceptions.HTTPError, requests.ConnectionError)
    Out[5]: False

    In [6]: issubclass(requests.ConnectionError, requests.exceptions.HTTPError)
    Out[6]: False

    In [8]: requests.ConnectionError is requests.exceptions.HTTPError
    Out[8]: False
---
 scripts/create_singularities | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/create_singularities b/scripts/create_singularities
index c1aa0904..51aef0bb 100755
--- a/scripts/create_singularities
+++ b/scripts/create_singularities
@@ -459,7 +459,7 @@ def retry_get(url: str) -> requests.Response:
     while True:
         try:
             return requests.get(url)
-        except (requests.ConnectionError, requests.Timeout) as e:
+        except (requests.ConnectionError, requests.exceptions.HTTPError, requests.Timeout) as e:
             if (wait := next(sleepiter, None)) is not None:
                 log.warning(
                     "Request to %s failed due to %s: %s; sleeping for %f"

From 60f7146ef7084849bc395a9354f2272cb5b06cc5 Mon Sep 17 00:00:00 2001
From: "John T. Wodder II" <git@varonathe.org>
Date: Tue, 20 Feb 2024 14:41:22 -0500
Subject: [PATCH 2/2] Correctly retry on 5xx responses

---
 scripts/create_singularities | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/scripts/create_singularities b/scripts/create_singularities
index 51aef0bb..85590484 100755
--- a/scripts/create_singularities
+++ b/scripts/create_singularities
@@ -80,7 +80,6 @@ class Builder:
     @staticmethod
     def get_last_docker_version_tag(dh: str, only_good_versions: bool=False) -> Optional[tuple[str, str]]:
         r = retry_get(f"https://registry.hub.docker.com/v2/repositories/{dh}/tags")
-        r.raise_for_status()
         versions = [cast(str, res["name"]) for res in r.json()["results"]]
         if len(versions) > 1 or (versions and only_good_versions):
             # select only the ones which seems to be semantic and/or
@@ -112,7 +111,6 @@ class Builder:
         """Return repositories for a specific namespace (user or organization)
         """
         r = retry_get(f"https://registry.hub.docker.com/v2/repositories/{namespace}")
-        r.raise_for_status()
         for res in r.json()["results"]:
             if res["repository_type"] != "image":
                 # don't know what to do with those
@@ -458,8 +456,15 @@ def retry_get(url: str) -> requests.Response:
     sleepiter = exp_wait(attempts=10)
     while True:
         try:
-            return requests.get(url)
-        except (requests.ConnectionError, requests.exceptions.HTTPError, requests.Timeout) as e:
+            r = requests.get(url)
+            r.raise_for_status()
+            return r
+        except (requests.ConnectionError, requests.HTTPError, requests.Timeout) as e:
+            if (
+                isinstance(e, requests.HTTPError)
+                and e.response is not None and e.response.status_code < 500
+            ):
+                raise e
             if (wait := next(sleepiter, None)) is not None:
                 log.warning(
                     "Request to %s failed due to %s: %s; sleeping for %f"
@@ -470,7 +475,7 @@ def retry_get(url: str) -> requests.Response:
                 )
                 sleep(wait)
             else:
-                raise
+                raise e
 
 
 def exp_wait(