chore: introduce MAX_JOB_ATTEMPTS

opengovsg · May 6, 2024 · 96a0a4b · 96a0a4b
1 parent 220a07c
commit 96a0a4b
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 1 deletion.
diff --git a/ecs/env.json b/ecs/env.json
@@ -165,6 +165,10 @@
     {
       "name": "M365_STEPS_LIMIT_PER_SEC",
       "valueFrom": "plumber-<ENVIRONMENT>-m365-steps-limit-per-sec"
+    },
+    {
+      "name": "MAX_JOB_ATTEMPTS",
+      "valueFrom": "plumber-<ENVIRONMENT>-max-job-attempts"
     }
   ]
 }
diff --git a/packages/backend/.env-example b/packages/backend/.env-example
@@ -33,6 +33,7 @@ M365_SG_GOVT_CLIENT_THUMBPRINT=...
 M365_SG_GOVT_CLIENT_PRIVATE_KEY=...
 M365_SG_GOVT_ALLOWED_SENSITIVITY_LABEL_GUIDS_CSV=11111111-1111-1111-1111-111111111111
 LAUNCH_DARKLY_SDK_KEY=...
+MAX_JOB_ATTEMPTS=3
 # LOCAL DEV
 SERVE_WEB_APP_SEPARATELY=true
 CLOUDFLARE_TOKEN=...

diff --git a/packages/backend/src/config/app.ts b/packages/backend/src/config/app.ts
@@ -43,6 +43,7 @@ type AppConfig = {
     privateKey: string
   }
   launchDarklySdkKey: string
+  maxJobAttempts: number
 }
 
 const port = process.env.PORT || '3000'
@@ -100,6 +101,7 @@ const appConfig: AppConfig = {
     rateLimit: parseInt(process.env.POSTMAN_RATE_LIMIT) || 169,
   },
   launchDarklySdkKey: process.env.LAUNCH_DARKLY_SDK_KEY,
+  maxJobAttempts: Number(process.env.MAX_JOB_ATTEMPTS ?? '10'),
 }
 
 if (!appConfig.encryptionKey) {
@@ -127,6 +129,15 @@ if (!appConfig.launchDarklySdkKey) {
   throw new Error('LAUNCH_DARKLY_SDK_KEY environment variable needs to be set!')
 }
 
+if (
+  isNaN(appConfig.maxJobAttempts) ||
+  !Number.isInteger(appConfig.maxJobAttempts)
+) {
+  throw new Error(
+    'MAX_JOB_ATTEMPTS environment variable is not a valid integer!',
+  )
+}
+
 // Force SGT date-time formatting no matter what
 LuxonSettings.defaultZone = 'Asia/Singapore'
 LuxonSettings.defaultLocale = 'en-SG'

diff --git a/packages/backend/src/helpers/default-job-configuration.ts b/packages/backend/src/helpers/default-job-configuration.ts
@@ -1,5 +1,7 @@
 import { type JobsProOptions } from '@taskforcesh/bullmq-pro'
 
+import appConfig from '@/config/app'
+
 export const REMOVE_AFTER_30_DAYS = {
   age: 30 * 24 * 3600,
 }
@@ -23,7 +25,7 @@ export const DEFAULT_JOB_DELAY_DURATION = 0
 //   steps to make progress. So in the worst case, a step might be retried
 //   log2(100) times ~= 7.
 // - We round that up to 10 just in case.
-export const MAXIMUM_JOB_ATTEMPTS = 10
+export const MAXIMUM_JOB_ATTEMPTS = appConfig.maxJobAttempts
 
 export const DEFAULT_JOB_OPTIONS: JobsProOptions = {
   removeOnComplete: REMOVE_AFTER_7_DAYS_OR_50_JOBS,