Merge pull request #242 from lsst-sqre/tickets/DM-43198

tickets/DM-43198: prevent eternal hangs during Slack refresh

pyproject.toml

@@ -29,8 +29,8 @@ name = "Association of Universities for Research in Astronomy, Inc. (AURA)"
 email = "[email protected]"
 
 [project.urls]
-Source = "https://github.com/lsst-sqre/gafaelfawr"
-"Issue tracker" = "https://github.com/lsst-sqre/gafaelfawr/issues"
+Source = "https://github.com/lsst-sqre/checkerboard"
+"Issue tracker" = "https://github.com/lsst-sqre/checkerboard/issues"
 
 [build-system]
 requires = [

src/checkerboard/storage/slack.py

@@ -300,16 +300,22 @@ async def _get_user_profile_from_slack(
         """Get a user profile.  Slack client will handle rate-limiting."""
         max_retries = 5
         retries = 0
-        while True:
+        last_exc: ClientConnectionError | TimeoutError | None = None
+        while retries < max_retries:
             try:
-                return await self._slack_client.users_profile_get(
-                    user=slack_id
-                )
-            except ClientConnectionError as exc:
-                if retries >= max_retries:
-                    raise
+                async with asyncio.timeout(60):
+                    # I don't know why we aren't timing out already.
+                    return await self._slack_client.users_profile_get(
+                        user=slack_id
+                    )
+            except (TimeoutError, ClientConnectionError) as exc:
                 await self._random_delay(f"Cannot connect to Slack: {exc}")
+                last_exc = exc
                 retries += 1
+        if last_exc is not None:
+            raise last_exc
+        # We should not get here; it will bubble up as a 500 if we do.
+        raise RuntimeError(f"Could not get Slack profile for {slack_id}")
 
     async def _random_delay(self, reason: str) -> None:
         """Delay for a random period between 2 and 5 (integral) seconds