fix: oom kill while stringifying large response json (#2754)

rudderlabs · Oct 26, 2023 · c8baf5b · c8baf5b
1 parent 52c432c
commit c8baf5b
Show file tree

Hide file tree

Showing 7 changed files with 1,268 additions and 1,171 deletions.
diff --git a/src/types/index.ts b/src/types/index.ts
@@ -137,7 +137,7 @@ type SourceTransformationResponse = {
 type DeliveryResponse = {
   status: number;
   message: string;
-  destinationResponse: object;
+  destinationResponse: any;
   statTags: object;
   authErrorCategory?: string;
 };

diff --git a/src/v0/util/index.js b/src/v0/util/index.js
@@ -1473,13 +1473,22 @@ const getErrorStatusCode = (error, defaultStatusCode = HTTP_STATUS_CODES.INTERNA
 /**
  * Used for generating error response with stats from native and built errors
  */
-function generateErrorObject(error, defTags = {}, shouldEnrichErrorMessage = false) {
-  let errObject = error;
+function generateErrorObject(error, defTags = {}, shouldEnrichErrorMessage = true) {
+  let errObject = new BaseError(
+    error.message,
+    getErrorStatusCode(error),
+    {
+      ...error.statTags,
+      ...defTags,
+    },
+    error.destinationResponse,
+    error.authErrorCategory,
+  );
   let errorMessage = error.message;
   if (shouldEnrichErrorMessage) {
     if (error.destinationResponse) {
       errorMessage = JSON.stringify({
-        message: error.message,
+        message: errorMessage,
         destinationResponse: error.destinationResponse,
       });
     }
@@ -1488,13 +1497,6 @@ function generateErrorObject(error, defTags = {}, shouldEnrichErrorMessage = fal
   if (!(error instanceof BaseError)) {
     errObject = new TransformationError(errorMessage, getErrorStatusCode(error));
   }
-
-  // Add higher level default tags
-  errObject.statTags = {
-    ...errObject.statTags,
-    ...defTags,
-  };
-
   return errObject;
 }
 /**

diff --git a/test/__tests__/data/marketo_router_output.json b/test/__tests__/data/marketo_router_output.json
@@ -328,7 +328,7 @@
   {
     "batched": false,
     "statusCode": 400,
-    "error": "Request Failed for marketo, Lookup field 'userId' not found (Aborted).[Marketo Transformer]: During lead look up using email",
+    "error": "{\"message\":\"Request Failed for marketo, Lookup field 'userId' not found (Aborted).[Marketo Transformer]: During lead look up using email\",\"destinationResponse\":{\"response\":{\"requestId\":\"142e4#1835b117b76\",\"success\":false,\"errors\":[{\"code\":\"1006\",\"message\":\"Lookup field 'userId' not found\"}]},\"status\":200}}",
     "statTags": {
       "errorCategory": "network",
       "errorType": "aborted"
@@ -479,7 +479,7 @@
   {
     "batched": false,
     "statusCode": 400,
-    "error": "Error occurred [Marketo Transformer]: During lead look up using email -> some other problem",
+    "error": "{\"message\":\"Error occurred [Marketo Transformer]: During lead look up using email -> some other problem\",\"destinationResponse\":{\"response\":{\"requestId\":\"142e4#1835b117b76\",\"success\":false,\"errors\":[{\"code\":\"random_marketo_code\",\"message\":\"some other problem\"}]},\"status\":200}}",
     "statTags": {
       "errorCategory": "network",
       "errorType": "aborted",