From 1ab783777c494d8a7f2969e55f35140e289d8794 Mon Sep 17 00:00:00 2001
From: Ian Chen <ian-@gmx.com>
Date: Mon, 15 Jan 2024 17:12:02 +0800
Subject: [PATCH] fix: increase telegram timeout and max attempts

---
 .../common/interceptor/rate-limit.ts          | 32 +++++++++++++++++++
 .../backend/src/apps/telegram-bot/index.ts    |  2 ++
 .../src/helpers/__tests__/backoff.test.ts     |  1 +
 packages/backend/src/helpers/backoff.ts       | 10 ++++--
 .../src/helpers/default-job-configuration.ts  |  2 +-
 5 files changed, 44 insertions(+), 3 deletions(-)
 create mode 100644 packages/backend/src/apps/telegram-bot/common/interceptor/rate-limit.ts

diff --git a/packages/backend/src/apps/telegram-bot/common/interceptor/rate-limit.ts b/packages/backend/src/apps/telegram-bot/common/interceptor/rate-limit.ts
new file mode 100644
index 000000000..00a61b60e
--- /dev/null
+++ b/packages/backend/src/apps/telegram-bot/common/interceptor/rate-limit.ts
@@ -0,0 +1,32 @@
+import type { IApp } from '@plumber/types'
+
+import logger from '@/helpers/logger'
+
+const rateLimitHandler: IApp['requestErrorHandler'] = async function (
+  $,
+  error,
+) {
+  if (error.response.status !== 429) {
+    return // use default error handling
+  }
+
+  const retryAfter = +error.response.headers['retry-after']
+  if (isNaN(retryAfter)) {
+    return
+  }
+
+  logger.error('Telegram rate limit error', {
+    executionId: $.execution.id,
+    flowId: $.flow.id,
+    stepId: $.step.id,
+    data: error.response.data,
+    headers: error.response.headers,
+    oldRetryAfter: retryAfter,
+    newRetryAfter: retryAfter + 30,
+  })
+
+  error.response.headers['retry-after'] = `${retryAfter + 30}`
+  throw error
+}
+
+export default rateLimitHandler
diff --git a/packages/backend/src/apps/telegram-bot/index.ts b/packages/backend/src/apps/telegram-bot/index.ts
index 9cd1f8bea..07093f922 100644
--- a/packages/backend/src/apps/telegram-bot/index.ts
+++ b/packages/backend/src/apps/telegram-bot/index.ts
@@ -1,6 +1,7 @@
 import { IApp } from '@plumber/types'
 
 import addAuthHeader from './common/add-auth-header'
+import rateLimitHandler from './common/interceptor/rate-limit'
 import actions from './actions'
 import auth from './auth'
 import dynamicData from './dynamic-data'
@@ -14,6 +15,7 @@ const app: IApp = {
   apiBaseUrl: 'https://api.telegram.org',
   primaryColor: '2AABEE',
   beforeRequest: [addAuthHeader],
+  requestErrorHandler: rateLimitHandler,
   dynamicData,
   auth,
   actions,
diff --git a/packages/backend/src/helpers/__tests__/backoff.test.ts b/packages/backend/src/helpers/__tests__/backoff.test.ts
index 2f8419b0e..69ebc5741 100644
--- a/packages/backend/src/helpers/__tests__/backoff.test.ts
+++ b/packages/backend/src/helpers/__tests__/backoff.test.ts
@@ -11,6 +11,7 @@ const mocks = vi.hoisted(() => ({
 vi.mock('@/helpers/logger', () => ({
   default: {
     error: mocks.logError,
+    info: vi.fn(),
   },
 }))
 
diff --git a/packages/backend/src/helpers/backoff.ts b/packages/backend/src/helpers/backoff.ts
index 3a7be505f..a4b1ff72c 100644
--- a/packages/backend/src/helpers/backoff.ts
+++ b/packages/backend/src/helpers/backoff.ts
@@ -25,7 +25,7 @@ export const exponentialBackoffWithJitter: BackoffStrategy = function (
   attemptsMade: number,
   _type: string,
   err: Error,
-  _job: Job,
+  job: Job,
 ): number {
   // This implements FullJitter-like jitter, with the following changes:
   //
@@ -40,5 +40,11 @@ export const exponentialBackoffWithJitter: BackoffStrategy = function (
   const initialDelay = computeInitialDelay(err)
 
   const prevFullDelay = Math.pow(2, attemptsMade - 1) * initialDelay
-  return prevFullDelay + Math.round(Math.random() * prevFullDelay)
+  const totalDelay = prevFullDelay + Math.round(Math.random() * prevFullDelay)
+  logger.info('Job delay calculation', {
+    ...job?.data,
+    attemptsMade,
+    delay: totalDelay,
+  })
+  return totalDelay
 }
diff --git a/packages/backend/src/helpers/default-job-configuration.ts b/packages/backend/src/helpers/default-job-configuration.ts
index 8008a2cdb..0aaca7482 100644
--- a/packages/backend/src/helpers/default-job-configuration.ts
+++ b/packages/backend/src/helpers/default-job-configuration.ts
@@ -10,7 +10,7 @@ export const REMOVE_AFTER_7_DAYS_OR_50_JOBS = {
 }
 
 export const DEFAULT_JOB_DELAY_DURATION = 0
-export const MAXIMUM_JOB_ATTEMPTS = 4
+export const MAXIMUM_JOB_ATTEMPTS = 6
 
 export const DEFAULT_JOB_OPTIONS: JobsOptions = {
   removeOnComplete: REMOVE_AFTER_7_DAYS_OR_50_JOBS,