fix: network handler fix

rudderlabs · Mar 21, 2024 · bd706f8 · bd706f8
1 parent bfb8a67
commit bd706f8
Show file tree

Hide file tree

Showing 5 changed files with 289 additions and 215 deletions.
diff --git a/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml b/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml
@@ -33,35 +33,10 @@ steps:
       })[]
 
   - name: batchSuccessfulEvents
-    description: Batches the successfulEvents using endpoint
-    condition: $.outputs.successfulEvents.length
+    description: Batches the successfulEvents
     template: |
-      let batches = $.batchEvents($.outputs.successfulEvents);
-      batches@batch.({
-        "batchedRequest": {
-          "body": {
-            "JSON": batch.message.body.JSON,
-            "JSON_ARRAY": {},
-            "XML": {},
-            "FORM": {}
-          },
-          "version": "1",
-          "type": "REST",
-          "method": "POST",
-          "endpoint": batch.message.endpoint,
-          "headers": batch.message.headers,
-          "params": {},
-          "files": {}
-        },
-        "metadata": batch.metadata,
-        "batched": true,
-        "statusCode": 200,
-        "destination": batch.destination
-      })[];
-    else:
-      name: returnEmptyOuput
-      template: '[]'
+      $.batchResponseBuilder($.outputs.successfulEvents);
 
   - name: finalPayload
     template: |
-      [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents]
+      [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents]
diff --git a/src/cdk/v2/destinations/linkedin_ads/utils.js b/src/cdk/v2/destinations/linkedin_ads/utils.js
@@ -107,34 +107,6 @@ const deduceConversionRules = (trackEventName, destConfig) => {
 
 const createConversionString = (ruleId) => `urn:lla:llaPartnerConversion:${ruleId}`;
 
-const batchEventChunks = (eventChunks) => {
-  const batchedEvents = [];
-  if (Array.isArray(eventChunks)) {
-    eventChunks.forEach((chunk) => {
-      const response = { destination: chunk[0].destination };
-      chunk.forEach((event, index) => {
-        if (index === 0) {
-          const [firstMessage] = event.message;
-          response.message = firstMessage;
-          response.destination = event.destination;
-          response.metadata = [event.metadata];
-        } else {
-          response.message.body.JSON.elements.push(...event.message[0].body.JSON.elements);
-          response.metadata.push(event.metadata);
-        }
-      });
-      batchedEvents.push(response);
-    });
-  }
-  return batchedEvents;
-};
-
-const batchEvents = (successfulEvents) => {
-  const eventChunks = lodash.chunk(successfulEvents, MAX_BATCH_SIZE);
-  const batchedEvents = batchEventChunks(eventChunks);
-  return batchedEvents;
-};
-
 const generateHeader = (accessToken) => {
   const headers = {
     'Content-Type': 'application/json',
@@ -166,14 +138,54 @@ const fetchAndVerifyConversionHappenedAt = (message) => {
   return timeInMilliseconds;
 };
 
+function batchResponseBuilder(successfulEvents) {
+  const constants = {
+    version: successfulEvents[0].message[0].version,
+    type: successfulEvents[0].message[0].type,
+    method: successfulEvents[0].message[0].method,
+    endpoint: successfulEvents[0].message[0].endpoint,
+    headers: successfulEvents[0].message[0].headers,
+    destination: successfulEvents[0].destination,
+  };
+
+  const allElements = successfulEvents.flatMap((event) => event.message[0].body.JSON.elements);
+  const allMetadata = successfulEvents.map((event) => event.metadata);
+
+  // Using lodash to chunk the elements into groups of up to 3
+  const chunkedElements = lodash.chunk(allElements, MAX_BATCH_SIZE);
+  const chunkedMetadata = lodash.chunk(allMetadata, MAX_BATCH_SIZE);
+
+  return chunkedElements.map((elementsBatch, index) => ({
+    batchedRequest: {
+      body: {
+        JSON: { elements: elementsBatch },
+        JSON_ARRAY: {},
+        XML: {},
+        FORM: {},
+      },
+      version: constants.version,
+      type: constants.type,
+      method: constants.method,
+      endpoint: constants.endpoint,
+      headers: constants.headers,
+      params: {},
+      files: {},
+    },
+    metadata: chunkedMetadata[index],
+    batched: true,
+    statusCode: 200,
+    destination: constants.destination,
+  }));
+}
+
 module.exports = {
   formatEmail,
   calculateConversionObject,
   curateUserInfoObject,
   fetchUserIds,
   deduceConversionRules,
   createConversionString,
-  batchEvents,
   generateHeader,
   fetchAndVerifyConversionHappenedAt,
+  batchResponseBuilder,
 };
diff --git a/src/v1/destinations/linkedin_ads/networkHandler.js b/src/v1/destinations/linkedin_ads/networkHandler.js
@@ -13,7 +13,6 @@ function constructPartialStatus(errorMessage) {
   let match;
   const errorMap = {};
 
-  // Note the added parentheses around the assignment
   // eslint-disable-next-line no-cond-assign
   while ((match = errorPattern.exec(errorMessage)) !== null) {
     const [, index, message] = match;
@@ -24,8 +23,12 @@ function constructPartialStatus(errorMessage) {
 }
 
 function createResponseArray(metadata, partialStatus) {
+  const partialStatusArray = Object.entries(partialStatus).map(([index, message]) => [
+    Number(index),
+    message,
+  ]);
   // Convert destPartialStatus to an object for easier lookup
-  const errorMap = partialStatus.reduce((acc, [index, message]) => {
+  const errorMap = partialStatusArray.reduce((acc, [index, message]) => {
     const jobId = metadata[index]?.jobId; // Get the jobId from the metadata array based on the index
     if (jobId !== undefined) {
       acc[jobId] = message;
@@ -72,7 +75,7 @@ const responseHandler = (responseParams) => {
     }
     // if the status is 422, we need to parse the error message and construct the response array
     if (status === 422) {
-      const destPartialStatus = constructPartialStatus(response.error?.message);
+      const destPartialStatus = constructPartialStatus(response?.message);
       responseWithIndividualEvents = [...createResponseArray(rudderJobMetadata, destPartialStatus)];
       return {
         status,

diff --git a/test/integrations/component.test.ts b/test/integrations/component.test.ts
@@ -133,14 +133,15 @@ const testRoute = async (route, tcData: TestCaseData) => {
   if (tcData.feature === tags.FEATURES.BATCH || tcData.feature === tags.FEATURES.ROUTER) {
     //TODO get rid of these skipped destinations after they are fixed
     if (
-      tcData.name != 'marketo_static_list' &&
-      tcData.name != 'mailmodo' &&
-      tcData.name != 'hs' &&
-      tcData.name != 'iterable' &&
-      tcData.name != 'klaviyo' &&
-      tcData.name != 'tiktok_ads' &&
-      tcData.name != 'mailjet' &&
-      tcData.name != 'google_adwords_offline_conversions'
+      (tcData.name != 'marketo_static_list' &&
+        tcData.name != 'mailmodo' &&
+        tcData.name != 'hs' &&
+        tcData.name != 'iterable' &&
+        tcData.name != 'klaviyo' &&
+        tcData.name != 'tiktok_ads' &&
+        tcData.name != 'mailjet' &&
+        tcData.name != 'google_adwords_offline_conversions',
+      tcData.name != 'linkedin_ads')
     ) {
       assertRouterOutput(response.body.output, tcData.input.request.body.input);
     }