Avoid getting stuck in infinite loop trying to refund prizes (#6080)

open-chat-labs · Jul 20, 2024 · 8d53b25 · 8d53b25
1 parent b17a5ba
commit 8d53b25
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 30 deletions.
diff --git a/backend/canisters/community/CHANGELOG.md b/backend/canisters/community/CHANGELOG.md
@@ -10,6 +10,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix fee then retry transfer if fee too high ([#6063](https://github.com/open-chat-labs/open-chat/pull/6063))
 - Handle transfer fee changing in either direction ([#6064](https://github.com/open-chat-labs/open-chat/pull/6064))
 
+### Fixed
+
+- Avoid getting stuck in infinite loop trying to refund prizes ([#6080](https://github.com/open-chat-labs/open-chat/pull/6080))
+
 ## [[2.0.1244](https://github.com/open-chat-labs/open-chat/releases/tag/v2.0.1244-community)] - 2024-07-18
 
 ### Added

diff --git a/backend/canisters/community/impl/src/timer_job_types.rs b/backend/canisters/community/impl/src/timer_job_types.rs
@@ -78,6 +78,8 @@ pub struct RefundPrizeJob {
 #[derive(Serialize, Deserialize, Clone)]
 pub struct MakeTransferJob {
     pub pending_transaction: PendingCryptoTransaction,
+    #[serde(default)]
+    pub attempt: u32,
 }
 
 #[derive(Serialize, Deserialize, Clone)]
@@ -202,7 +204,10 @@ impl Job for HardDeleteMessageContentJob {
                                 if let Some(pending_transaction) =
                                     prize.prize_refund(sender, &MEMO_PRIZE_REFUND, state.env.now_nanos())
                                 {
-                                    follow_on_jobs.push(TimerJob::MakeTransfer(MakeTransferJob { pending_transaction }));
+                                    follow_on_jobs.push(TimerJob::MakeTransfer(MakeTransferJob {
+                                        pending_transaction,
+                                        attempt: 0,
+                                    }));
                                 }
                             }
                         }
@@ -277,7 +282,10 @@ impl Job for RefundPrizeJob {
                 None
             }
         }) {
-            let make_transfer_job = MakeTransferJob { pending_transaction };
+            let make_transfer_job = MakeTransferJob {
+                pending_transaction,
+                attempt: 0,
+            };
             make_transfer_job.execute();
         }
     }
@@ -287,22 +295,27 @@ impl Job for MakeTransferJob {
     fn execute(self) {
         let sender = read_state(|state| state.env.canister_id());
         let pending = self.pending_transaction;
-        ic_cdk::spawn(make_transfer(pending, sender));
+        ic_cdk::spawn(make_transfer(pending, sender, self.attempt));
 
-        async fn make_transfer(mut pending_transaction: PendingCryptoTransaction, sender: CanisterId) {
+        async fn make_transfer(mut pending_transaction: PendingCryptoTransaction, sender: CanisterId, attempt: u32) {
             if let Err(error) = process_transaction(pending_transaction.clone(), sender, true).await {
                 error!(?error, "Transaction failed");
-                mutate_state(|state| {
-                    let now = state.env.now();
-                    if (pending_transaction.created() / NANOS_PER_MILLISECOND) + DAY_IN_MS < now {
-                        pending_transaction.set_created(now * NANOS_PER_MILLISECOND);
-                    }
-                    state.data.timer_jobs.enqueue_job(
-                        TimerJob::MakeTransfer(MakeTransferJob { pending_transaction }),
-                        now + MINUTE_IN_MS,
-                        now,
-                    );
-                });
+                if attempt < 50 {
+                    mutate_state(|state| {
+                        let now = state.env.now();
+                        if (pending_transaction.created() / NANOS_PER_MILLISECOND) + DAY_IN_MS < now {
+                            pending_transaction.set_created(now * NANOS_PER_MILLISECOND);
+                        }
+                        state.data.timer_jobs.enqueue_job(
+                            TimerJob::MakeTransfer(MakeTransferJob {
+                                pending_transaction,
+                                attempt: attempt + 1,
+                            }),
+                            now + MINUTE_IN_MS,
+                            now,
+                        );
+                    });
+                }
             }
         }
     }

diff --git a/backend/canisters/group/CHANGELOG.md b/backend/canisters/group/CHANGELOG.md
@@ -10,6 +10,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix fee then retry transfer if fee too high ([#6063](https://github.com/open-chat-labs/open-chat/pull/6063))
 - Handle transfer fee changing in either direction ([#6064](https://github.com/open-chat-labs/open-chat/pull/6064))
 
+### Fixed
+
+- Avoid getting stuck in infinite loop trying to refund prizes ([#6080](https://github.com/open-chat-labs/open-chat/pull/6080))
+
 ## [[2.0.1245](https://github.com/open-chat-labs/open-chat/releases/tag/v2.0.1245-group)] - 2024-07-18
 
 ### Added

diff --git a/backend/canisters/group/impl/src/timer_job_types.rs b/backend/canisters/group/impl/src/timer_job_types.rs
@@ -49,6 +49,8 @@ pub struct RefundPrizeJob {
 #[derive(Serialize, Deserialize, Clone)]
 pub struct MakeTransferJob {
     pub pending_transaction: PendingCryptoTransaction,
+    #[serde(default)]
+    pub attempt: u32,
 }
 
 #[derive(Serialize, Deserialize, Clone)]
@@ -171,7 +173,10 @@ impl Job for HardDeleteMessageContentJob {
                                 if let Some(pending_transaction) =
                                     prize.prize_refund(sender, &MEMO_PRIZE_REFUND, state.env.now_nanos())
                                 {
-                                    follow_on_jobs.push(TimerJob::MakeTransfer(MakeTransferJob { pending_transaction }));
+                                    follow_on_jobs.push(TimerJob::MakeTransfer(MakeTransferJob {
+                                        pending_transaction,
+                                        attempt: 0,
+                                    }));
                                 }
                             }
                         }
@@ -226,7 +231,10 @@ impl Job for RefundPrizeJob {
                 state.env.now_nanos(),
             )
         }) {
-            let make_transfer_job = MakeTransferJob { pending_transaction };
+            let make_transfer_job = MakeTransferJob {
+                pending_transaction,
+                attempt: 0,
+            };
             make_transfer_job.execute();
         }
     }
@@ -236,22 +244,27 @@ impl Job for MakeTransferJob {
     fn execute(self) {
         let sender = read_state(|state| state.env.canister_id());
         let pending = self.pending_transaction.clone();
-        ic_cdk::spawn(make_transfer(pending, sender));
+        ic_cdk::spawn(make_transfer(pending, sender, self.attempt));
 
-        async fn make_transfer(mut pending_transaction: PendingCryptoTransaction, sender: CanisterId) {
+        async fn make_transfer(mut pending_transaction: PendingCryptoTransaction, sender: CanisterId, attempt: u32) {
             if let Err(error) = process_transaction(pending_transaction.clone(), sender, true).await {
                 error!(?error, "Transaction failed");
-                mutate_state(|state| {
-                    let now = state.env.now();
-                    if (pending_transaction.created() / NANOS_PER_MILLISECOND) + DAY_IN_MS < now {
-                        pending_transaction.set_created(now * NANOS_PER_MILLISECOND);
-                    }
-                    state.data.timer_jobs.enqueue_job(
-                        TimerJob::MakeTransfer(MakeTransferJob { pending_transaction }),
-                        now + MINUTE_IN_MS,
-                        now,
-                    );
-                });
+                if attempt < 50 {
+                    mutate_state(|state| {
+                        let now = state.env.now();
+                        if (pending_transaction.created() / NANOS_PER_MILLISECOND) + DAY_IN_MS < now {
+                            pending_transaction.set_created(now * NANOS_PER_MILLISECOND);
+                        }
+                        state.data.timer_jobs.enqueue_job(
+                            TimerJob::MakeTransfer(MakeTransferJob {
+                                pending_transaction,
+                                attempt: attempt + 1,
+                            }),
+                            now + MINUTE_IN_MS,
+                            now,
+                        );
+                    });
+                }
             }
         }
     }