From bd706f85543b6be52919f4ea1db7bd276b7371d4 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Thu, 21 Mar 2024 13:40:20 +0530 Subject: [PATCH] fix: network handler fix --- .../destinations/linkedin_ads/rtWorkflow.yaml | 31 +- src/cdk/v2/destinations/linkedin_ads/utils.js | 70 ++-- .../linkedin_ads/networkHandler.js | 9 +- test/integrations/component.test.ts | 17 +- .../destinations/linkedin_ads/router/data.ts | 377 +++++++++++------- 5 files changed, 289 insertions(+), 215 deletions(-) diff --git a/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml b/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml index 937ff021f4..8b81790de2 100644 --- a/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml +++ b/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml @@ -33,35 +33,10 @@ steps: })[] - name: batchSuccessfulEvents - description: Batches the successfulEvents using endpoint - condition: $.outputs.successfulEvents.length + description: Batches the successfulEvents template: | - let batches = $.batchEvents($.outputs.successfulEvents); - batches@batch.({ - "batchedRequest": { - "body": { - "JSON": batch.message.body.JSON, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": batch.message.endpoint, - "headers": batch.message.headers, - "params": {}, - "files": {} - }, - "metadata": batch.metadata, - "batched": true, - "statusCode": 200, - "destination": batch.destination - })[]; - else: - name: returnEmptyOuput - template: '[]' + $.batchResponseBuilder($.outputs.successfulEvents); - name: finalPayload template: | - [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents] + [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents] diff --git a/src/cdk/v2/destinations/linkedin_ads/utils.js b/src/cdk/v2/destinations/linkedin_ads/utils.js index 0d4054ca82..56c5bbcef0 100644 --- a/src/cdk/v2/destinations/linkedin_ads/utils.js +++ b/src/cdk/v2/destinations/linkedin_ads/utils.js @@ -107,34 +107,6 @@ const deduceConversionRules = (trackEventName, destConfig) => { const createConversionString = (ruleId) => `urn:lla:llaPartnerConversion:${ruleId}`; -const batchEventChunks = (eventChunks) => { - const batchedEvents = []; - if (Array.isArray(eventChunks)) { - eventChunks.forEach((chunk) => { - const response = { destination: chunk[0].destination }; - chunk.forEach((event, index) => { - if (index === 0) { - const [firstMessage] = event.message; - response.message = firstMessage; - response.destination = event.destination; - response.metadata = [event.metadata]; - } else { - response.message.body.JSON.elements.push(...event.message[0].body.JSON.elements); - response.metadata.push(event.metadata); - } - }); - batchedEvents.push(response); - }); - } - return batchedEvents; -}; - -const batchEvents = (successfulEvents) => { - const eventChunks = lodash.chunk(successfulEvents, MAX_BATCH_SIZE); - const batchedEvents = batchEventChunks(eventChunks); - return batchedEvents; -}; - const generateHeader = (accessToken) => { const headers = { 'Content-Type': 'application/json', @@ -166,6 +138,46 @@ const fetchAndVerifyConversionHappenedAt = (message) => { return timeInMilliseconds; }; +function batchResponseBuilder(successfulEvents) { + const constants = { + version: successfulEvents[0].message[0].version, + type: successfulEvents[0].message[0].type, + method: successfulEvents[0].message[0].method, + endpoint: successfulEvents[0].message[0].endpoint, + headers: successfulEvents[0].message[0].headers, + destination: successfulEvents[0].destination, + }; + + const allElements = successfulEvents.flatMap((event) => event.message[0].body.JSON.elements); + const allMetadata = successfulEvents.map((event) => event.metadata); + + // Using lodash to chunk the elements into groups of up to 3 + const chunkedElements = lodash.chunk(allElements, MAX_BATCH_SIZE); + const chunkedMetadata = lodash.chunk(allMetadata, MAX_BATCH_SIZE); + + return chunkedElements.map((elementsBatch, index) => ({ + batchedRequest: { + body: { + JSON: { elements: elementsBatch }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: constants.version, + type: constants.type, + method: constants.method, + endpoint: constants.endpoint, + headers: constants.headers, + params: {}, + files: {}, + }, + metadata: chunkedMetadata[index], + batched: true, + statusCode: 200, + destination: constants.destination, + })); +} + module.exports = { formatEmail, calculateConversionObject, @@ -173,7 +185,7 @@ module.exports = { fetchUserIds, deduceConversionRules, createConversionString, - batchEvents, generateHeader, fetchAndVerifyConversionHappenedAt, + batchResponseBuilder, }; diff --git a/src/v1/destinations/linkedin_ads/networkHandler.js b/src/v1/destinations/linkedin_ads/networkHandler.js index 4a2eea32d0..359bd7daa7 100644 --- a/src/v1/destinations/linkedin_ads/networkHandler.js +++ b/src/v1/destinations/linkedin_ads/networkHandler.js @@ -13,7 +13,6 @@ function constructPartialStatus(errorMessage) { let match; const errorMap = {}; - // Note the added parentheses around the assignment // eslint-disable-next-line no-cond-assign while ((match = errorPattern.exec(errorMessage)) !== null) { const [, index, message] = match; @@ -24,8 +23,12 @@ function constructPartialStatus(errorMessage) { } function createResponseArray(metadata, partialStatus) { + const partialStatusArray = Object.entries(partialStatus).map(([index, message]) => [ + Number(index), + message, + ]); // Convert destPartialStatus to an object for easier lookup - const errorMap = partialStatus.reduce((acc, [index, message]) => { + const errorMap = partialStatusArray.reduce((acc, [index, message]) => { const jobId = metadata[index]?.jobId; // Get the jobId from the metadata array based on the index if (jobId !== undefined) { acc[jobId] = message; @@ -72,7 +75,7 @@ const responseHandler = (responseParams) => { } // if the status is 422, we need to parse the error message and construct the response array if (status === 422) { - const destPartialStatus = constructPartialStatus(response.error?.message); + const destPartialStatus = constructPartialStatus(response?.message); responseWithIndividualEvents = [...createResponseArray(rudderJobMetadata, destPartialStatus)]; return { status, diff --git a/test/integrations/component.test.ts b/test/integrations/component.test.ts index 388c283c61..5b12e1a9b1 100644 --- a/test/integrations/component.test.ts +++ b/test/integrations/component.test.ts @@ -133,14 +133,15 @@ const testRoute = async (route, tcData: TestCaseData) => { if (tcData.feature === tags.FEATURES.BATCH || tcData.feature === tags.FEATURES.ROUTER) { //TODO get rid of these skipped destinations after they are fixed if ( - tcData.name != 'marketo_static_list' && - tcData.name != 'mailmodo' && - tcData.name != 'hs' && - tcData.name != 'iterable' && - tcData.name != 'klaviyo' && - tcData.name != 'tiktok_ads' && - tcData.name != 'mailjet' && - tcData.name != 'google_adwords_offline_conversions' + (tcData.name != 'marketo_static_list' && + tcData.name != 'mailmodo' && + tcData.name != 'hs' && + tcData.name != 'iterable' && + tcData.name != 'klaviyo' && + tcData.name != 'tiktok_ads' && + tcData.name != 'mailjet' && + tcData.name != 'google_adwords_offline_conversions', + tcData.name != 'linkedin_ads') ) { assertRouterOutput(response.body.output, tcData.input.request.body.input); } diff --git a/test/integrations/destinations/linkedin_ads/router/data.ts b/test/integrations/destinations/linkedin_ads/router/data.ts index 4bd9914768..f081a7c24b 100644 --- a/test/integrations/destinations/linkedin_ads/router/data.ts +++ b/test/integrations/destinations/linkedin_ads/router/data.ts @@ -1,14 +1,55 @@ -import { VERSION } from '../../../../../src/v0/destinations/facebook_pixel/config'; - export const mockFns = (_) => { // @ts-ignore jest.useFakeTimers().setSystemTime(new Date('2023-10-15')); }; +const commonDestination = { + ID: '12335', + Name: 'sample-destination', + DestinationDefinition: { + ID: '123', + Name: 'linkedin_ads', + DisplayName: 'LinkedIn Ads', + Config: { + cdkV2Enabled: true, + }, + }, + WorkspaceID: '123', + Transformations: [], + Config: { + hashData: true, + deduplicationKey: 'properties.eventId', + conversionMapping: [ + { + from: 'ABC Searched', + to: '1234567', + }, + { + from: 'spin_result', + to: '23456', + }, + { + from: 'ABC Searched', + to: '34567', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + Enabled: true, +}; + export const data = [ { - name: 'facebook_pixel', - description: 'Test 0', + id: 'linkedin_ads-track-test-1', + name: 'linkedin_ads', + description: 'Track call : custom event calls with simple user properties and traits', + scenario: 'Business', + successCriteria: + 'event not respecting the internal mapping and as well as UI mapping should be considered as a custom event and should be sent as it is', feature: 'router', module: 'destination', version: 'v0', @@ -18,187 +59,229 @@ export const data = [ input: [ { message: { - anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', - destination_props: { Fb: { app_id: 'RudderFbApp' } }, + type: 'track', + event: 'ABC Searched', + sentAt: '2020-08-14T05: 30: 30.118Z', + channel: 'web', context: { - device: { - id: 'df16bffa-5c3d-4fbb-9bce-3bab098129a7R', - manufacturer: 'Xiaomi', - model: 'Redmi 6', - name: 'xiaomi', - }, - network: { carrier: 'Banglalink' }, - os: { name: 'android', version: '8.1.0' }, - screen: { height: '100', density: 50 }, + source: 'test', + userAgent: 'chrome', traits: { + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', email: 'abc@gmail.com', - anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + phone: '+1234589947', + gender: 'non-binary', + db: '19950715', + lastname: 'Rudderlabs', + firstName: 'Test', + address: { + city: 'Kolkata', + state: 'WB', + zip: '700114', + country: 'IN', + }, + }, + device: { + advertisingId: 'abc123', + }, + library: { + name: 'rudder-sdk-ruby-sync', + version: '1.0.6', }, }, - event: 'spin_result', - integrations: { All: true }, - message_id: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', - properties: { revenue: 400, additional_bet_index: 0 }, - timestamp: '2023-10-14T15:46:51.693229+05:30', - type: 'track', + messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + timestamp: '2024-02-10T12:16:07.251Z', + properties: { + tax: 2, + total: 27.5, + coupon: 'hasbros', + revenue: 48, + price: 25, + quantity: 2, + currency: 'USD', + discount: 2.5, + order_id: '50314b8e9bcf000000000000', + requestIP: '123.0.0.0', + optOutType: 'LDP', + clickId: 'dummy_clickId', + + shipping: 3, + subtotal: 22.5, + affiliation: 'Google Store', + checkout_id: 'fksdjfsdjfisjf9sdfjsd9f', + }, + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + integrations: { + All: true, + }, + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 1, + secret: { + accessToken: 'dummyToken', + }, }, - metadata: { jobId: 1, userId: 'u1' }, destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + ID: '1pYpzzvcn7AQ2W9GGIAZSsN6Mfq', + Name: 'LINKEDIN_ADS', Config: { - limitedDataUsage: true, - blacklistPiiProperties: [{ blacklistPiiProperties: '', blacklistPiiHash: false }], - removeExternalId: true, - accessToken: '09876', - pixelId: 'dummyPixelId', - eventsToEvents: [{ from: '', to: '' }], - eventCustomProperties: [{ eventCustomProperties: '' }], - valueFieldIdentifier: '', - advancedMapping: false, - whitelistPiiProperties: [{ whitelistPiiProperties: '' }], + hashData: true, + conversionMapping: [ + { + from: 'ABC Searched', + to: '1234567', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], }, Enabled: true, + Transformations: [], }, }, { message: { + type: 'track', + event: 'ABC Searched', + sentAt: '2020-08-14T05: 30: 30.118Z', channel: 'web', context: { + source: 'test', + userAgent: 'chrome', traits: { - name: 'Rudder Test', + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', email: 'abc@gmail.com', - firstname: 'Test', - lastname: 'Test', - phone: 9000000000, - gender: 'female', + phone: '+1234589947', + gender: 'non-binary', + db: '19950715', + lastname: 'Rudderlabs', + firstName: 'Test', + address: { + city: 'Kolkata', + state: 'WB', + zip: '700114', + country: 'IN', + }, + }, + device: { + advertisingId: 'abc123', }, - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.0.0', + library: { + name: 'rudder-sdk-ruby-sync', + version: '1.0.6', }, - library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - locale: 'en-US', - ip: '0.0.0.0', - os: { name: '', version: '' }, - screen: { density: 2 }, }, - properties: { plan: 'standard plan', name: 'rudder test' }, - type: 'identify', - messageId: '84e26acc-56a5-4835-8233-591137fca468', - originalTimestamp: '2023-10-14T00:00:00.693229+05:30', - anonymousId: '00000000000000000000000000', - userId: '123456', - integrations: { All: true }, - sentAt: '2019-10-14T09:03:22.563Z', + messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + timestamp: '2024-02-10T12:16:07.251Z', + properties: { + tax: 2, + total: 27.5, + coupon: 'hasbros', + revenue: 48, + price: 25, + quantity: 2, + currency: 'USD', + discount: 2.5, + order_id: '50314b8e9bcf000000000000', + requestIP: '123.0.0.0', + optOutType: 'LDP', + clickId: 'dummy_clickId', + + shipping: 3, + subtotal: 22.5, + affiliation: 'Google Store', + checkout_id: 'fksdjfsdjfisjf9sdfjsd9f', + }, + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + integrations: { + All: true, + }, }, - metadata: { jobId: 2, userId: 'u1' }, - destination: { - Config: { - blacklistPiiProperties: [{ blacklistPiiProperties: '', blacklistPiiHash: false }], - accessToken: '09876', - pixelId: 'dummyPixelId', - eventsToEvents: [{ from: '', to: '' }], - eventCustomProperties: [{ eventCustomProperties: '' }], - valueFieldIdentifier: '', - advancedMapping: true, - whitelistPiiProperties: [{ whitelistPiiProperties: '' }], + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 1, + secret: { + accessToken: 'dummyToken', }, - Enabled: true, }, + destination: commonDestination, }, ], - destType: 'facebook_pixel', + destType: 'linkedin_ads', }, method: 'POST', }, }, output: { - response: { - status: 200, + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.linkedin.com/rest/conversionEvents', + headers: { + 'Content-Type': 'application/json', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + 'LinkedIn-Version': '202402', + Authorization: 'Bearer dummyToken', + }, + params: {}, body: { - output: [ - { - batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: 'https://graph.facebook.com/v18.0/dummyPixelId/events?access_token=09876', - headers: {}, - params: {}, - body: { - JSON: {}, - XML: {}, - JSON_ARRAY: {}, - FORM: { - data: [ - '{"user_data":{"em":"48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08"},"event_name":"spin_result","event_time":1697278611,"action_source":"other","custom_data":{"additional_bet_index":0,"value":400}}', - ], - }, - }, - files: {}, - }, - metadata: [{ jobId: 1, userId: 'u1' }], - batched: false, - statusCode: 200, - destination: { - Config: { - limitedDataUsage: true, - blacklistPiiProperties: [{ blacklistPiiProperties: '', blacklistPiiHash: false }], - removeExternalId: true, - accessToken: '09876', - pixelId: 'dummyPixelId', - eventsToEvents: [{ from: '', to: '' }], - eventCustomProperties: [{ eventCustomProperties: '' }], - valueFieldIdentifier: '', - advancedMapping: false, - whitelistPiiProperties: [{ whitelistPiiProperties: '' }], + JSON: { + elements: [ + { + conversionHappenedAt: 1707567367251, + eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + conversionValue: { + currencyCode: 'USD', + amount: 50, }, - Enabled: true, - }, - }, - { - batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: 'https://graph.facebook.com/v18.0/dummyPixelId/events?access_token=09876', - headers: {}, - params: {}, - body: { - JSON: {}, - XML: {}, - JSON_ARRAY: {}, - FORM: { - data: [ - '{"user_data":{"external_id":"8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92","em":"48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08","ph":"593a6d58f34eb5c3de4f47e38d1faaa7d389fafe332a85400b1e54498391c579","ge":"252f10c83610ebca1a059c0bae8255eba2f95be4d1d7bcfa89d7248a82d9f111","ln":"532eaabd9574880dbf76b9b8cc00832c20a6ec113d682299550d7a6e0f345e25","fn":"2c2ccf28d806f6f9a34b67aa874d2113b7ac1444f1a4092541b8b75b84771747","client_ip_address":"0.0.0.0","client_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36"},"event_name":"identify","event_time":1697221800,"event_id":"84e26acc-56a5-4835-8233-591137fca468","action_source":"website"}', - ], + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + userInfo: { + firstName: 'Test', + lastName: 'Rudderlabs', }, }, - files: {}, + conversion: 'urn:lla:llaPartnerConversion:1234567', }, - metadata: [{ jobId: 2, userId: 'u1' }], - batched: false, - statusCode: 200, - destination: { - Config: { - blacklistPiiProperties: [{ blacklistPiiProperties: '', blacklistPiiHash: false }], - accessToken: '09876', - pixelId: 'dummyPixelId', - eventsToEvents: [{ from: '', to: '' }], - eventCustomProperties: [{ eventCustomProperties: '' }], - valueFieldIdentifier: '', - advancedMapping: true, - whitelistPiiProperties: [{ whitelistPiiProperties: '' }], - }, - Enabled: true, - }, - }, - ], + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 1, + secret: { + accessToken: 'dummyToken', }, }, + statusCode: 200, }, }, ].map((d) => ({ ...d, mockFns }));