From f86906305f9ea48c31eb729aa909004bb8d6123e Mon Sep 17 00:00:00 2001
From: Martin Malina <mmalina@redhat.com>
Date: Wed, 25 Sep 2024 13:11:59 +0200
Subject: [PATCH] [RELEASE-1117] Add retry when requesting RADAS signature

We run the signing pipeline in Konflux a lot and users
report that oftentimes one or a few of the requests fail.

In theory some retry should work on the Python library
level, but it doesn't really seem to work. So this should
help. See Jira for more details.

Signed-off-by: Martin Malina <mmalina@redhat.com>
---
 .../openshift/tasks/request-signature.yml     | 40 +++++++++++++------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/ansible/roles/operator-pipeline/templates/openshift/tasks/request-signature.yml b/ansible/roles/operator-pipeline/templates/openshift/tasks/request-signature.yml
index f83e759d6..353b3d711 100644
--- a/ansible/roles/operator-pipeline/templates/openshift/tasks/request-signature.yml
+++ b/ansible/roles/operator-pipeline/templates/openshift/tasks/request-signature.yml
@@ -62,19 +62,35 @@ spec:
         #! /usr/bin/env bash
         set -xe
 
+        MAX_RETRIES=3
+        RETRY_DELAY=5 # Initial delay
+
         echo "Requesting signing from RADAS"
-        request-signature \
-          --manifest-digest "$(params.manifest_digest)" \
-          --output signing_response.json \
-          --reference "$(params.reference)" \
-          --requester "$(params.requester)" \
-          --sig-key-id "$(params.sig_key_id)" \
-          --sig-key-name "$(params.sig_key_name)" \
-          --umb-client-name "$(params.umb_client_name)" \
-          --umb-listen-topic "$(params.umb_listen_topic)" \
-          --umb-publish-topic "$(params.umb_publish_topic)" \
-          --umb-url "$(params.umb_url)" \
-          --verbose
+        for ((i=1; i<=MAX_RETRIES; i++)); do
+          if request-signature \
+            --manifest-digest "$(params.manifest_digest)" \
+            --output signing_response.json \
+            --reference "$(params.reference)" \
+            --requester "$(params.requester)" \
+            --sig-key-id "$(params.sig_key_id)" \
+            --sig-key-name "$(params.sig_key_name)" \
+            --umb-client-name "$(params.umb_client_name)" \
+            --umb-listen-topic "$(params.umb_listen_topic)" \
+            --umb-publish-topic "$(params.umb_publish_topic)" \
+            --umb-url "$(params.umb_url)" \
+            --verbose
+          then
+            echo "request-signature command succeeded."
+            break
+          elif [ $i -eq $MAX_RETRIES ]; then
+            echo "Max retries reached. Exiting."
+            exit 1
+          else
+            echo "Attempt $i failed. Retrying in $RETRY_DELAY seconds..."
+            sleep $RETRY_DELAY
+            RETRY_DELAY=$((RETRY_DELAY * 2))  # Exponential backoff
+          fi
+        done
 
         SIG_DATA=$(cat signing_response.json)
         echo "Signed claims and their metadata: "