From f44a10533fe49888f8254e29795c3b9368201788 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 25 Sep 2023 13:49:17 +0530 Subject: [PATCH 01/19] fix: timestamp microseconds input cm360 --- .../data/CampaignManagerTrackConfig.json | 6 +- .../campaign_manager/transform.js | 51 ++- test/__tests__/data/campaign_manager.json | 351 +++++++++++++++++- .../data/campaign_manager_router_output.json | 4 +- 4 files changed, 365 insertions(+), 47 deletions(-) diff --git a/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json b/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json index e547bb0f35..f36ba8da8a 100644 --- a/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json +++ b/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json @@ -12,11 +12,7 @@ { "destKey": "timestampMicros", "sourceKeys": "timestamp", - "sourceFromGenericMap": true, - "required": true, - "metadata": { - "type": "microSecondTimestamp" - } + "sourceFromGenericMap": true }, { "destKey": "floodlightActivityId", diff --git a/src/v0/destinations/campaign_manager/transform.js b/src/v0/destinations/campaign_manager/transform.js index 99e5fe9c57..723980f0e4 100644 --- a/src/v0/destinations/campaign_manager/transform.js +++ b/src/v0/destinations/campaign_manager/transform.js @@ -1,5 +1,4 @@ const { EventType } = require('../../../constants'); - const { constructPayload, defaultRequestConfig, @@ -72,7 +71,17 @@ function processTrack(message, metadata, destination) { delete requestJson.childDirectedTreatment; delete requestJson.limitAdTracking; } - requestJson.timestampMicros = requestJson.timestampMicros.toString(); + + const date = new Date(requestJson.timestampMicros); + let unixTimestamp = date.getTime(); + // Date, moment both are not able to distinguish input if it is second,millisecond or microsecond unix timestamp + // Using count of digits to distinguish between the 3, 9999999999999 (13 digits) means Nov 20 2286 + if (unixTimestamp.toString().length === 13) { // milliseconds + unixTimestamp *= 1000; + } else if (unixTimestamp.toString().length === 10) { // seconds + unixTimestamp *= 1000000; + } + requestJson.timestampMicros = unixTimestamp.toString(); const encryptionInfo = {}; // prepare encrptionInfo if encryptedUserId or encryptedUserIdCandidates is given @@ -138,35 +147,17 @@ function postValidateRequest(response) { ); } - let count = 0; - - if (response.body.JSON.conversions[0].gclid) { - count += 1; - } - - if (response.body.JSON.conversions[0].dclid) { - count += 1; - } - - if (response.body.JSON.conversions[0].encryptedUserId) { - count += 1; - } - - if (response.body.JSON.conversions[0].encryptedUserIdCandidates) { - count += 1; - } - - if (response.body.JSON.conversions[0].mobileDeviceId) { - count += 1; - } - - if (response.body.JSON.conversions[0].impressionId) { - count += 1; - } - - if (count !== 1) { + if ( + !response.body.JSON.conversions[0].gclid && + !response.body.JSON.conversions[0].matchId && + !response.body.JSON.conversions[0].dclid && + !response.body.JSON.conversions[0].encryptedUserId && + !response.body.JSON.conversions[0].encryptedUserIdCandidates && + !response.body.JSON.conversions[0].mobileDeviceId && + !response.body.JSON.conversions[0].impressionId + ) { throw new InstrumentationError( - '[CAMPAIGN MANAGER (DCM)]: For CM360 we need one of encryptedUserId,encryptedUserIdCandidates, matchId, mobileDeviceId, gclid, dclid, impressionId.', + '[CAMPAIGN MANAGER (DCM)]: Atleast one of encryptedUserId,encryptedUserIdCandidates, matchId, mobileDeviceId, gclid, dclid, impressionId.', ); } } diff --git a/test/__tests__/data/campaign_manager.json b/test/__tests__/data/campaign_manager.json index ac5b363f64..6139ddbeb3 100644 --- a/test/__tests__/data/campaign_manager.json +++ b/test/__tests__/data/campaign_manager.json @@ -46,8 +46,6 @@ "density": 2 } }, - "event": "Promotion Clicked", - "type": "track", "originalTimestamp": "2022-11-17T00:22:02.903+05:30", "properties": { "profileId": "34245", @@ -112,7 +110,7 @@ { "floodlightConfigurationId": "213123123", "ordinal": "string", - "timestampMicros": "1668624722000000", + "timestampMicros": "1668624722903000", "floodlightActivityId": "456543345245", "quantity": "455678", "value": 756, @@ -178,9 +176,7 @@ "density": 2 } }, - "event": "Promotion Clicked", - "type": "track", - "originalTimestamp": "2022-11-17T00:22:02.903+05:30", + "originalTimestamp": "2020-01-24T11:59:02.402+05:30", "properties": { "profileId": "34245", "floodlightConfigurationId": "213123123", @@ -238,7 +234,7 @@ { "floodlightConfigurationId": "213123123", "ordinal": "string", - "timestampMicros": "1668624722000000", + "timestampMicros": "1579847342402000", "floodlightActivityId": "456543345245", "quantity": "455678", "value": 756, @@ -256,7 +252,7 @@ } }, { - "description": "Track - batch insert Call", + "description": "Track - batch insert Call failure", "input": { "message": { "channel": "web", @@ -302,8 +298,6 @@ "density": 2 } }, - "event": "Promotion Clicked", - "type": "track", "originalTimestamp": "2022-11-17T00:22:02.903+05:30", "properties": { "profileId": "34245", @@ -356,5 +350,342 @@ "scope": "exception" } } + }, + { + "description": "Track - batch update Call timestamp as unix", + "input": { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "device": { + "id": "0572f78fa49c648e", + "name": "generic_x86_arm", + "type": "Android", + "model": "AOSP on IA Emulator", + "manufacturer": "Google", + "adTrackingEnabled": true, + "advertisingId": "44c97318-9040-4361-8bc7-4eb30f665ca8" + }, + "traits": { + "email": "alex@example.com", + "phone": "+1-202-555-0146", + "firstName": "John", + "lastName": "Gomes", + "city": "London", + "state": "England", + "countryCode": "GB", + "postalCode": "EC3M", + "streetAddress": "71 Cherry Court SOUTHAMPTON SO53 5PD UK" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, + "properties": { + "profileId": "34245", + "floodlightConfigurationId": "213123123", + "ordinal": "string", + "floodlightActivityId": "456543345245", + "value": "756", + "quantity": "455678", + "timestamp": "1668624722903000", + "matchId": "12312321", + "encryptionSource": "AD_SERVING", + "encryptionEntityId": "3564523", + "encryptionEntityType": "DCM_ACCOUNT", + "requestType": "batchupdate" + }, + "originalTimestamp": 1668624722555000, + "type": "track", + "event": "event test", + "anonymousId": "randomId", + "integrations": { + "All": true + }, + "name": "ApplicationLoaded", + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "secret": { + "access_token": "dummyApiToken", + "refresh_token": "efgh5678", + "developer_token": "ijkl91011" + } + }, + "destination": { + "Config": { + "profileId": "5343234", + "treatmentForUnderage": false, + "limitAdTracking": false, + "childDirectedTreatment": false, + "nonPersonalizedAd": false, + "rudderAccountId": "2EOknn1JNH7WK1MfNku4fGYKkRK" + } + } + }, + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate", + "headers": { + "Authorization": "Bearer dummyApiToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "kind": "dfareporting#conversionsBatchUpdateRequest", + "conversions": [ + { + "floodlightConfigurationId": "213123123", + "ordinal": "string", + "timestampMicros": "1668624722555000", + "floodlightActivityId": "456543345245", + "quantity": "455678", + "value": 756, + "matchId": "12312321", + "treatmentForUnderage": false, + "nonPersonalizedAd": false + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + }, + { + "description": "Track - batch insert Call failure", + "input": { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "device": { + "id": "0572f78fa49c648e", + "name": "generic_x86_arm", + "type": "Android", + "model": "AOSP on IA Emulator", + "manufacturer": "Google", + "adTrackingEnabled": true, + "advertisingId": "44c97318-9040-4361-8bc7-4eb30f665ca8" + }, + "traits": { + "email": "alex@example.com", + "phone": "+1-202-555-0146", + "firstName": "John", + "lastName": "Gomes", + "city": "London", + "state": "England", + "countryCode": "GB", + "postalCode": "EC3M", + "streetAddress": "71 Cherry Court SOUTHAMPTON SO53 5PD UK" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, + "originalTimestamp": "2022-11-17T00:22:02.903+05:30", + "properties": { + "profileId": "34245", + "floodlightConfigurationId": "213123123", + "ordinal": "string", + "floodlightActivityId": "456543345245", + "value": "756", + "quantity": "455678", + "requestType": "batchinsert" + }, + "type": "track", + "event": "event test", + "anonymousId": "randomId", + "integrations": { + "All": true + }, + "name": "ApplicationLoaded", + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "secret": { + "access_token": "dummyApiToken", + "refresh_token": "efgh5678", + "developer_token": "ijkl91011" + } + }, + "destination": { + "Config": { + "profileId": "5343234", + "treatmentForUnderage": false, + "limitAdTracking": false, + "childDirectedTreatment": false, + "nonPersonalizedAd": false, + "rudderAccountId": "2EOknn1JNH7WK1MfNku4fGYKkRK" + } + } + }, + "output": { + "statusCode": 400, + "error": "[CAMPAIGN MANAGER (DCM)]: Atleast one of encryptedUserId,encryptedUserIdCandidates, matchId, mobileDeviceId, gclid, dclid, impressionId.", + "statTags": { + "destination": "blueshift", + "stage": "transform", + "scope": "exception" + } + } + }, + { + "description": "Track - batch update Call timestamp as unix", + "input": { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "device": { + "id": "0572f78fa49c648e", + "name": "generic_x86_arm", + "type": "Android", + "model": "AOSP on IA Emulator", + "manufacturer": "Google", + "adTrackingEnabled": true, + "advertisingId": "44c97318-9040-4361-8bc7-4eb30f665ca8" + }, + "traits": { + "email": "alex@example.com", + "phone": "+1-202-555-0146", + "firstName": "John", + "lastName": "Gomes", + "city": "London", + "state": "England", + "countryCode": "GB", + "postalCode": "EC3M", + "streetAddress": "71 Cherry Court SOUTHAMPTON SO53 5PD UK" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, + "properties": { + "profileId": "34245", + "floodlightConfigurationId": "213123123", + "ordinal": "string", + "floodlightActivityId": "456543345245", + "value": "756", + "quantity": "455678", + "timestamp": "1668624722903000", + "matchId": "12312321", + "requestType": "batchupdate" + }, + "originalTimestamp": 1668624722555000, + "type": "track", + "event": "event test", + "anonymousId": "randomId", + "integrations": { + "All": true + }, + "name": "ApplicationLoaded", + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "secret": { + "access_token": "dummyApiToken", + "refresh_token": "efgh5678", + "developer_token": "ijkl91011" + } + }, + "destination": { + "Config": { + "profileId": "5343234", + "treatmentForUnderage": false, + "limitAdTracking": false, + "childDirectedTreatment": false, + "nonPersonalizedAd": false, + "rudderAccountId": "2EOknn1JNH7WK1MfNku4fGYKkRK" + } + } + }, + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate", + "headers": { + "Authorization": "Bearer dummyApiToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "kind": "dfareporting#conversionsBatchUpdateRequest", + "conversions": [ + { + "floodlightConfigurationId": "213123123", + "ordinal": "string", + "timestampMicros": "1668624722555000", + "floodlightActivityId": "456543345245", + "quantity": "455678", + "value": 756, + "matchId": "12312321", + "treatmentForUnderage": false, + "nonPersonalizedAd": false + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } } ] diff --git a/test/__tests__/data/campaign_manager_router_output.json b/test/__tests__/data/campaign_manager_router_output.json index e3a9a39c28..61c3438ab6 100644 --- a/test/__tests__/data/campaign_manager_router_output.json +++ b/test/__tests__/data/campaign_manager_router_output.json @@ -23,7 +23,7 @@ { "nonPersonalizedAd": false, "treatmentForUnderage": false, - "timestampMicros": "1668624722000000", + "timestampMicros": "1668624722903000", "floodlightConfigurationId": "213123123", "ordinal": "string", "quantity": "455678", @@ -87,7 +87,7 @@ { "nonPersonalizedAd": false, "treatmentForUnderage": false, - "timestampMicros": "1668624722000000", + "timestampMicros": "1668624722903000", "floodlightConfigurationId": "213123123", "ordinal": "string", "quantity": "455678", From 3771b3dc24f9a5d8c5e5e1a781c823e923b810dd Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 25 Sep 2023 17:58:17 +0530 Subject: [PATCH 02/19] feat: batching in cm360 --- .../destinations/campaign_manager/config.js | 3 + .../campaign_manager/transform.js | 122 +++++++++++++++++- test/__tests__/campaign_manager.test.js | 5 +- .../data/campaign_manager_router_input.json | 94 +++++++++++++- .../data/campaign_manager_router_output.json | 27 +++- 5 files changed, 232 insertions(+), 19 deletions(-) diff --git a/src/v0/destinations/campaign_manager/config.js b/src/v0/destinations/campaign_manager/config.js index 063f65cb14..b3a9531347 100644 --- a/src/v0/destinations/campaign_manager/config.js +++ b/src/v0/destinations/campaign_manager/config.js @@ -9,6 +9,8 @@ const ConfigCategories = { }, }; +const MAX_BATCH_CONVERSATIONS_SIZE = 1000; + const EncryptionEntityType = [ 'ENCRYPTION_ENTITY_TYPE_UNKNOWN', 'DCM_ACCOUNT', @@ -28,4 +30,5 @@ module.exports = { BASE_URL, EncryptionEntityType, EncryptionSource, + MAX_BATCH_CONVERSATIONS_SIZE, }; diff --git a/src/v0/destinations/campaign_manager/transform.js b/src/v0/destinations/campaign_manager/transform.js index 723980f0e4..a8983ebccd 100644 --- a/src/v0/destinations/campaign_manager/transform.js +++ b/src/v0/destinations/campaign_manager/transform.js @@ -1,11 +1,15 @@ +const lodash = require('lodash'); const { EventType } = require('../../../constants'); const { constructPayload, defaultRequestConfig, defaultPostRequestConfig, + defaultBatchRequestConfig, removeUndefinedAndNullValues, + getSuccessRespEvents, isDefinedAndNotNull, - simpleProcessRouterDest, + checkInvalidRtTfEvents, + handleRtTfSingleEventError, getAccessToken, } = require('../../util'); @@ -15,6 +19,7 @@ const { BASE_URL, EncryptionEntityType, EncryptionSource, + MAX_BATCH_CONVERSATIONS_SIZE, } = require('./config'); const { InstrumentationError } = require('../../util/errorTypes'); @@ -75,10 +80,12 @@ function processTrack(message, metadata, destination) { const date = new Date(requestJson.timestampMicros); let unixTimestamp = date.getTime(); // Date, moment both are not able to distinguish input if it is second,millisecond or microsecond unix timestamp - // Using count of digits to distinguish between the 3, 9999999999999 (13 digits) means Nov 20 2286 - if (unixTimestamp.toString().length === 13) { // milliseconds + // Using count of digits to distinguish between the 3, 9999999999999 (13 digits) means Nov 20 2286 + if (unixTimestamp.toString().length === 13) { + // milliseconds unixTimestamp *= 1000; - } else if (unixTimestamp.toString().length === 10) { // seconds + } else if (unixTimestamp.toString().length === 10) { + // seconds unixTimestamp *= 1000000; } requestJson.timestampMicros = unixTimestamp.toString(); @@ -147,7 +154,7 @@ function postValidateRequest(response) { ); } - if ( + if ( !response.body.JSON.conversions[0].gclid && !response.body.JSON.conversions[0].matchId && !response.body.JSON.conversions[0].dclid && @@ -185,9 +192,110 @@ function process(event) { return response; } +const generateBatch = (eventKind, events) => { + const batchRequestObject = defaultBatchRequestConfig(); + const conversions = []; + let encryptionInfo = {}; + const metadata = []; + // extracting destination, message from the first event in a batch + const { destination, message } = events[0]; + // Batch event into dest batch structure + events.forEach((ev) => { + conversions.push(...ev.message.body.JSON.conversions); + metadata.push(ev.metadata); + if (ev.message.body.JSON.encryptionInfo) { + encryptionInfo = ev.message.body.JSON.encryptionInfo; + } + }); + + batchRequestObject.batchedRequest.body.JSON = { + kind: eventKind, + conversions, + }; + + if (Object.keys(encryptionInfo).length > 0) { + batchRequestObject.batchedRequest.body.JSON.encryptionInfo = encryptionInfo; + } + + batchRequestObject.batchedRequest.endpoint = message.endpoint; + + batchRequestObject.batchedRequest.headers = message.headers; + + return { + ...batchRequestObject, + metadata, + destination, + }; +}; + +const batchEvents = (eventChunksArray) => { + const batchedResponseList = []; + + // group batchInsert and batchUpdate payloads + const groupedEventChunks = lodash.groupBy( + eventChunksArray, + (event) => event.message.body.JSON.kind, + ); + Object.keys(groupedEventChunks).forEach((eventKind) => { + // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] + const eventChunks = lodash.chunk(groupedEventChunks[eventKind], MAX_BATCH_CONVERSATIONS_SIZE); + eventChunks.forEach((chunk) => { + const batchEventResponse = generateBatch(eventKind, chunk); + batchedResponseList.push( + getSuccessRespEvents( + batchEventResponse.batchedRequest, + batchEventResponse.metadata, + batchEventResponse.destination, + true, + ), + ); + }); + }); + return batchedResponseList; +}; + const processRouterDest = async (inputs, reqMetadata) => { - const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); - return respList; + const errorRespEvents = checkInvalidRtTfEvents(inputs); + if (errorRespEvents.length > 0) { + return errorRespEvents; + } + + const batchErrorRespList = []; + const eventChunksArray = []; + const { destination } = inputs[0]; + await Promise.all( + inputs.map(async (event) => { + try { + if (event.message.statusCode) { + // already transformed event + eventChunksArray.push({ + message: event.message, + metadata: event.metadata, + destination, + }); + } else { + // if not transformed + const proccessedRespList = process(event); + const transformedPayload = { + message: proccessedRespList, + metadata: event.metadata, + destination, + }; + eventChunksArray.push(transformedPayload); + } + } catch (error) { + const errRespEvent = handleRtTfSingleEventError(event, error, reqMetadata); + batchErrorRespList.push(errRespEvent); + } + }), + ); + + let batchResponseList = []; + if (eventChunksArray.length > 0) { + batchResponseList = batchEvents(eventChunksArray); + } + + return [...batchResponseList, ...batchErrorRespList]; }; module.exports = { process, processRouterDest }; diff --git a/test/__tests__/campaign_manager.test.js b/test/__tests__/campaign_manager.test.js index 4f3a40d520..15568eddb2 100644 --- a/test/__tests__/campaign_manager.test.js +++ b/test/__tests__/campaign_manager.test.js @@ -28,8 +28,7 @@ describe(`${name} Tests`, () => { testData.forEach((dataPoint, index) => { it(`${index}. ${integration} - ${dataPoint.description}`, async () => { try { - let output = await transformer.process(dataPoint.input); - delete output.body.JSON.idempotency; + const output = await transformer.process(dataPoint.input); expect(output).toEqual(dataPoint.output); } catch (error) { expect(error.message).toEqual(dataPoint.output.error); @@ -37,7 +36,7 @@ describe(`${name} Tests`, () => { }); }); }); - + describe("Router Tests", () => { it("Payload", async () => { const routerOutput = await transformer.processRouterDest(inputRouterData); diff --git a/test/__tests__/data/campaign_manager_router_input.json b/test/__tests__/data/campaign_manager_router_input.json index 18893a0b43..8cae574c55 100644 --- a/test/__tests__/data/campaign_manager_router_input.json +++ b/test/__tests__/data/campaign_manager_router_input.json @@ -61,8 +61,6 @@ "density": 2 } }, - "event": "Promotion Clicked", - "type": "track", "originalTimestamp": "2022-11-17T00:22:02.903+05:30", "properties": { "profileId": 437689, @@ -70,7 +68,7 @@ "ordinal": "string", "quantity": "455678", "floodlightActivityId": "456543345245", - "value": 7, + "value": 47, "encryptedUserIdCandidates": ["dfghjbnm"], "limitAdTracking": true, "childDirectedTreatment": true, @@ -151,8 +149,94 @@ "density": 2 } }, - "event": "Promotion Clicked", + "originalTimestamp": "2023-11-17T00:22:02.903+05:30", + "properties": { + "profileId": 437689, + "floodlightConfigurationId": "213123123", + "ordinal": "345345345", + "quantity": "4556784325345345", + "floodlightActivityId": "456543345245", + "value": 17, + "encryptedUserIdCandidates": ["dfghjbnm"], + "limitAdTracking": true, + "childDirectedTreatment": true, + "encryptionSource": "AD_SERVING", + "encryptionEntityId": "3564523", + "encryptionEntityType": "DCM_ACCOUNT", + "requestType": "batchinsert" + }, "type": "track", + "event": "event test", + "anonymousId": "randomId", + "integrations": { + "All": true + }, + "name": "ApplicationLoaded", + "sentAt": "2023-11-17T00:22:02.903+05:30" + } + }, + { + "metadata": { + "secret": { + "access_token": "dummyApiToken", + "refresh_token": "efgh5678", + "developer_token": "ijkl91011" + }, + "jobId": 2 + }, + "destination": { + "Config": { + "treatmentForUnderage": false, + "limitAdTracking": false, + "childDirectedTreatment": false, + "nonPersonalizedAd": false, + "rudderAccountId": "2EOknn1JNH7WK1MfNku4fGYKkRK" + } + }, + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "device": { + "id": "0572f78fa49c648e", + "name": "generic_x86_arm", + "type": "Android", + "model": "AOSP on IA Emulator", + "manufacturer": "Google", + "adTrackingEnabled": true, + "advertisingId": "44c97318-9040-4361-8bc7-4eb30f665ca8" + }, + "traits": { + "email": "alex@example.com", + "phone": "+1-202-555-0146", + "firstName": "John", + "lastName": "Gomes", + "city": "London", + "state": "England", + "countryCode": "GB", + "postalCode": "EC3M", + "streetAddress": "71 Cherry Court SOUTHAMPTON SO53 5PD UK" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, "originalTimestamp": "2022-11-17T00:22:02.903+05:30", "properties": { "profileId": 437689, @@ -241,8 +325,6 @@ "density": 2 } }, - "event": "Promotion Clicked", - "type": "track", "originalTimestamp": "2022-11-17T00:22:02.903+05:30", "properties": { "profileId": 437689, diff --git a/test/__tests__/data/campaign_manager_router_output.json b/test/__tests__/data/campaign_manager_router_output.json index 61c3438ab6..f92713e2b3 100644 --- a/test/__tests__/data/campaign_manager_router_output.json +++ b/test/__tests__/data/campaign_manager_router_output.json @@ -28,10 +28,23 @@ "ordinal": "string", "quantity": "455678", "floodlightActivityId": "456543345245", - "value": 7, + "value": 47, "encryptedUserIdCandidates": ["dfghjbnm"], "limitAdTracking": true, "childDirectedTreatment": true + }, + { + "floodlightConfigurationId": "213123123", + "ordinal": "345345345", + "quantity": "4556784325345345", + "floodlightActivityId": "456543345245", + "value": 17, + "encryptedUserIdCandidates": ["dfghjbnm"], + "limitAdTracking": true, + "childDirectedTreatment": true, + "nonPersonalizedAd": false, + "timestampMicros": "1700160722903000", + "treatmentForUnderage": false } ] }, @@ -49,9 +62,17 @@ "refresh_token": "efgh5678" }, "jobId": 1 + }, + { + "jobId": 2, + "secret": { + "access_token": "dummyApiToken", + "developer_token": "ijkl91011", + "refresh_token": "efgh5678" + } } ], - "batched": false, + "batched": true, "statusCode": 200, "destination": { "Config": { @@ -113,7 +134,7 @@ "jobId": 2 } ], - "batched": false, + "batched": true, "statusCode": 200, "destination": { "Config": { From 3e589e70e72e70436899b39e379d0419b1a034b6 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Tue, 26 Sep 2023 16:30:04 +0530 Subject: [PATCH 03/19] chore: addressed comments --- .../data/CampaignManagerTrackConfig.json | 3 +- .../campaign_manager/networkHandler.js | 34 ++++- .../campaign_manager/transform.js | 11 +- test/__tests__/data/campaign_manager.json | 126 +++++++++++------- 4 files changed, 121 insertions(+), 53 deletions(-) diff --git a/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json b/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json index f36ba8da8a..c8bd10d524 100644 --- a/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json +++ b/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json @@ -12,7 +12,8 @@ { "destKey": "timestampMicros", "sourceKeys": "timestamp", - "sourceFromGenericMap": true + "sourceFromGenericMap": true, + "required": true }, { "destKey": "floodlightActivityId", diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index a80fff6fe4..d8498b7621 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -9,14 +9,36 @@ const { AbortedError, RetryableError, NetworkError } = require('../../util/error const tags = require('../../util/tags'); function checkIfFailuresAreRetryable(response) { + const { status } = response; + let isRetryable = true; try { - if (Array.isArray(response.status) && Array.isArray(response.status[0].errors)) { - return ( - response.status[0].errors[0].code !== 'PERMISSION_DENIED' && - response.status[0].errors[0].code !== 'INVALID_ARGUMENT' - ); + if (Array.isArray(status)) { + // iterate over each status, and if found retryable in conversations ..retry else discard + /* status : [{ + "conversion": { + object (Conversion) + }, + "errors": [ + { + object (ConversionError) + } + ], + "kind": string + }] */ + for (const st of status) { + st.errors.forEach(err => { + // if code is any of these, event is not retryable + if ( + err.code === 'PERMISSION_DENIED' || + err.code === 'INVALID_ARGUMENT' || + err.code === 'NOT_FOUND' + ) { + isRetryable = false; + } + }) + } } - return true; + return isRetryable; } catch (e) { return true; } diff --git a/src/v0/destinations/campaign_manager/transform.js b/src/v0/destinations/campaign_manager/transform.js index a8983ebccd..5393f18631 100644 --- a/src/v0/destinations/campaign_manager/transform.js +++ b/src/v0/destinations/campaign_manager/transform.js @@ -77,10 +77,19 @@ function processTrack(message, metadata, destination) { delete requestJson.limitAdTracking; } + // for handling when input is timestamp as string + const numTimestamp = /^\d+$/.test(requestJson.timestampMicros); + if (numTimestamp) { + // is digit only, below convert string timestamp to numeric + requestJson.timestampMicros *= 1; + } + + // 2022-10-11T05:453:90.ZZ + // 16483423423423423 const date = new Date(requestJson.timestampMicros); let unixTimestamp = date.getTime(); // Date, moment both are not able to distinguish input if it is second,millisecond or microsecond unix timestamp - // Using count of digits to distinguish between the 3, 9999999999999 (13 digits) means Nov 20 2286 + // Using count of digits to distinguish between these 3, 9999999999999 (13 digits) means Nov 20 2286 which is long far in future if (unixTimestamp.toString().length === 13) { // milliseconds unixTimestamp *= 1000; diff --git a/test/__tests__/data/campaign_manager.json b/test/__tests__/data/campaign_manager.json index 6139ddbeb3..4347ce8394 100644 --- a/test/__tests__/data/campaign_manager.json +++ b/test/__tests__/data/campaign_manager.json @@ -572,53 +572,89 @@ "description": "Track - batch update Call timestamp as unix", "input": { "message": { - "channel": "web", - "context": { - "app": { - "build": "1.0.0", - "name": "RudderLabs JavaScript SDK", - "namespace": "com.rudderlabs.javascript", - "version": "1.0.0" - }, - "device": { - "id": "0572f78fa49c648e", - "name": "generic_x86_arm", - "type": "Android", - "model": "AOSP on IA Emulator", - "manufacturer": "Google", - "adTrackingEnabled": true, - "advertisingId": "44c97318-9040-4361-8bc7-4eb30f665ca8" - }, - "traits": { - "email": "alex@example.com", - "phone": "+1-202-555-0146", - "firstName": "John", - "lastName": "Gomes", - "city": "London", - "state": "England", - "countryCode": "GB", - "postalCode": "EC3M", - "streetAddress": "71 Cherry Court SOUTHAMPTON SO53 5PD UK" - }, - "library": { - "name": "RudderLabs JavaScript SDK", - "version": "1.0.0" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - "locale": "en-US", - "ip": "0.0.0.0", - "os": { - "name": "", - "version": "" - }, - "screen": { - "density": 2 - } + "channel": "sources", + "properties": { + "profileId": "34245", + "floodlightConfigurationId": "213123123", + "ordinal": "213123", + "floodlightActivityId": "456543345245", + "value": "756", + "quantity": "455678", + "timestamp": "1668624722903000", + "matchId": "12312321", + "requestType": "batchupdate" + }, + "timestamp": "1668624722555000", + "type": "track", + "event": "event test", + "anonymousId": "randomId", + "integrations": { + "All": true }, + "name": "ApplicationLoaded", + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "secret": { + "access_token": "dummyApiToken", + "refresh_token": "efgh5678", + "developer_token": "ijkl91011" + } + }, + "destination": { + "Config": { + "profileId": "5343234", + "treatmentForUnderage": false, + "limitAdTracking": false, + "childDirectedTreatment": false, + "nonPersonalizedAd": false, + "rudderAccountId": "2EOknn1JNH7WK1MfNku4fGYKkRK" + } + } + }, + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate", + "headers": { + "Authorization": "Bearer dummyApiToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "kind": "dfareporting#conversionsBatchUpdateRequest", + "conversions": [ + { + "floodlightConfigurationId": "213123123", + "ordinal": "213123", + "timestampMicros": "1668624722555000", + "floodlightActivityId": "456543345245", + "quantity": "455678", + "value": 756, + "matchId": "12312321", + "treatmentForUnderage": false, + "nonPersonalizedAd": false + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + }, + { + "description": "Track - batch update Call timestamp as unix as digit", + "input": { + "message": { + "channel": "sources", "properties": { "profileId": "34245", "floodlightConfigurationId": "213123123", - "ordinal": "string", + "ordinal": "213123", "floodlightActivityId": "456543345245", "value": "756", "quantity": "455678", @@ -626,7 +662,7 @@ "matchId": "12312321", "requestType": "batchupdate" }, - "originalTimestamp": 1668624722555000, + "timestamp": 1668624722555000, "type": "track", "event": "event test", "anonymousId": "randomId", @@ -670,7 +706,7 @@ "conversions": [ { "floodlightConfigurationId": "213123123", - "ordinal": "string", + "ordinal": "213123", "timestampMicros": "1668624722555000", "floodlightActivityId": "456543345245", "quantity": "455678", From 44f10483208c8e9e65d544de791ec33808f51582 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Tue, 26 Sep 2023 18:18:48 +0530 Subject: [PATCH 04/19] chore: addressed comments --- src/v0/destinations/campaign_manager/networkHandler.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index d8498b7621..d094567fb2 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -10,7 +10,6 @@ const tags = require('../../util/tags'); function checkIfFailuresAreRetryable(response) { const { status } = response; - let isRetryable = true; try { if (Array.isArray(status)) { // iterate over each status, and if found retryable in conversations ..retry else discard @@ -26,19 +25,19 @@ function checkIfFailuresAreRetryable(response) { "kind": string }] */ for (const st of status) { - st.errors.forEach(err => { + for(const err of st.errors) { // if code is any of these, event is not retryable if ( err.code === 'PERMISSION_DENIED' || err.code === 'INVALID_ARGUMENT' || err.code === 'NOT_FOUND' ) { - isRetryable = false; + return false; } - }) + } } } - return isRetryable; + return true; } catch (e) { return true; } From 0cf895e309ad032770b5ab010a124bc625f5f089 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Wed, 27 Sep 2023 02:36:27 +0530 Subject: [PATCH 05/19] fix: return aborted inside cath block --- src/v0/destinations/campaign_manager/networkHandler.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index d094567fb2..a8c57902e5 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -39,7 +39,7 @@ function checkIfFailuresAreRetryable(response) { } return true; } catch (e) { - return true; + return false; } } @@ -73,7 +73,7 @@ const responseHandler = (destinationResponse) => { } throw new NetworkError( - `Campaign Manager: ${response.error.message} during CAMPAIGN_MANAGER response transformation 3`, + `Campaign Manager: ${response.error?.message} during CAMPAIGN_MANAGER response transformation`, status, { [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), From eed94e784e52c64c232e89b84037fa2c0b8ee8d6 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 9 Oct 2023 17:01:31 +0530 Subject: [PATCH 06/19] feat: update batching partial events --- src/types/index.ts | 1 + .../campaign_manager/networkHandler.js | 220 ++++++++++++++++-- 2 files changed, 203 insertions(+), 18 deletions(-) diff --git a/src/types/index.ts b/src/types/index.ts index ff0c1f88b9..dd43158a24 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -136,6 +136,7 @@ type DeliveryResponse = { destinationResponse: object; statTags: object; authErrorCategory?: string; + response?: object; }; enum MessageType { diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index a8c57902e5..0fea5fefb7 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -1,3 +1,5 @@ +/* eslint-disable no-param-reassign */ +/* eslint-disable no-restricted-syntax */ const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); const { isHttpStatusSuccess, getAuthErrCategoryFromStCode } = require('../../util/index'); @@ -5,10 +7,11 @@ const { processAxiosResponse, getDynamicErrorType, } = require('../../../adapters/utils/networkUtils'); -const { AbortedError, RetryableError, NetworkError } = require('../../util/errorTypes'); +const { NetworkError } = require('../../util/errorTypes'); const tags = require('../../util/tags'); -function checkIfFailuresAreRetryable(response) { +// eslint-disable-next-line @typescript-eslint/no-unused-vars +function checkIfFailuresAreRetryable(response, proxyOutputObj) { const { status } = response; try { if (Array.isArray(status)) { @@ -43,35 +46,216 @@ function checkIfFailuresAreRetryable(response) { } } +function isEventRetryable(element, proxyOutputObj) { + let flag = false; + let errorMsg = ""; + // success event + if (!element.errors) { + return flag; + } + for(const err of element.errors) { + errorMsg += `${err.message}, `; + if (err.code === 'INTERNAL') { + flag = true; + } + } + if (errorMsg) { + proxyOutputObj.error = errorMsg; + } + return flag; +} + +function isEventAbortable(element, proxyOutputObj) { + let flag = false; + let errorMsg = ""; + // success event + if (!element.errors) { + return flag; + } + for(const err of element.errors) { + errorMsg += `${err.message}, `; + // if code is any of these, event is not retryable + if ( + err.code === 'PERMISSION_DENIED' || + err.code === 'INVALID_ARGUMENT' || + err.code === 'NOT_FOUND' + ) { + flag = true; + } + } + if (errorMsg) { + proxyOutputObj.error = errorMsg; + } + return flag; +} + const responseHandler = (destinationResponse) => { const message = `[CAMPAIGN_MANAGER Response Handler] - Request Processed Successfully`; - const { response, status } = destinationResponse; + const responseForPartialEventHandling = [] + destinationResponse = { + response: { + hasFailures: true, + status: [ + { + conversion: { + floodlightConfigurationId: "213123123", + floodlightActivityId: "456543345245", + timestampMicros: "1668624722555000", + value: 756234234234, + quantity: "3", + ordinal: "1", + limitAdTracking: false, + childDirectedTreatment: false, + gclid: "123", + nonPersonalizedAd: false, + treatmentForUnderage: false, + kind: "dfareporting#conversion", + }, + kind: "dfareporting#conversionStatus", + }, + { + conversion: { + floodlightConfigurationId: "213123123", + floodlightActivityId: "456543345245", + timestampMicros: "1668624722555000", + value: 756234234234, + quantity: "3", + ordinal: "1", + limitAdTracking: false, + childDirectedTreatment: false, + gclid: "123", + nonPersonalizedAd: false, + treatmentForUnderage: false, + kind: "dfareporting#conversion", + }, + errors: [ + { + code: "NOT_FOUND", + message: "Floodlight config id: 213123123 was not found.", + kind: "dfareporting#conversionError", + }, + ], + kind: "dfareporting#conversionStatus", + }, + { + conversion: { + floodlightConfigurationId: "213123123", + floodlightActivityId: "456543345245", + timestampMicros: "1668624722555000", + value: 756234234234, + quantity: "3", + ordinal: "1", + limitAdTracking: false, + childDirectedTreatment: false, + gclid: "123", + nonPersonalizedAd: false, + treatmentForUnderage: false, + kind: "dfareporting#conversion", + }, + errors: [ + { + code: "NOT_FOUND", + message: "Floodlight config id: 213123123 was not found.", + kind: "dfareporting#conversionError", + }, + ], + kind: "dfareporting#conversionStatus", + }, + ], + kind: "dfareporting#conversionsBatchInsertResponse", + }, + status: 200, + rudderJobMetadata: [{ + jobId: 10, + attemptNum: 0, + userId: "", + sourceId: "2Vsge2uWYdrLfG7pZb5Y82eo4lr", + destinationId: "2RHh08uOsXqE9KvCDg3hoaeuK2L", + workspaceId: "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + secret: { + access_token: "ya29.a0AfB_byCptyxhlxHnWGZWiV9MkQNpF_wm6ZF2DGVxgM1Ii-nEX0CRdKotqRvK19M5i8mwzjorOnbxL8Ix7zGdtkB-mGHLmTvjU7eJkzk2q_XnRbwsfIfjs2z5UfFCFcoZ5TUSDTp6qjX4hESYx6TWS9OqxgiqiApjPEb2EwaCgYKAcYSARMSFQGOcNnC23uomxEzJUCjzXVCfuqi2Q0173", + refresh_token: "1//0dk2dltGhN4n3CgYIARAAGA0SNwF-L9Irh9E6xb10sQaAjUfbdO1a7hx3bKnKze-_jK8OTHoowHwIynRPPshiFDeb3wtgqAnt9yI", + developer_token: "developer_Token", + }, + }, + { + jobId: 11, + attemptNum: 0, + userId: "", + sourceId: "2Vsge2uWYdrLfG7pZb5Y82eo4lr", + destinationId: "2RHh08uOsXqE9KvCDg3hoaeuK2L", + workspaceId: "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + secret: { + access_token: "ya29.a0AfB_byCptyxhlxHnWGZWiV9MkQNpF_wm6ZF2DGVxgM1Ii-nEX0CRdKotqRvK19M5i8mwzjorOnbxL8Ix7zGdtkB-mGHLmTvjU7eJkzk2q_XnRbwsfIfjs2z5UfFCFcoZ5TUSDTp6qjX4hESYx6TWS9OqxgiqiApjPEb2EwaCgYKAcYSARMSFQGOcNnC23uomxEzJUCjzXVCfuqi2Q0173", + refresh_token: "1//0dk2dltGhN4n3CgYIARAAGA0SNwF-L9Irh9E6xb10sQaAjUfbdO1a7hx3bKnKze-_jK8OTHoowHwIynRPPshiFDeb3wtgqAnt9yI", + developer_token: "developer_Token", + }, + }, + { + jobId: 12, + attemptNum: 0, + userId: "", + sourceId: "2Vsge2uWYdrLfG7pZb5Y82eo4lr", + destinationId: "2RHh08uOsXqE9KvCDg3hoaeuK2L", + workspaceId: "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + secret: { + access_token: "ya29.a0AfB_byCptyxhlxHnWGZWiV9MkQNpF_wm6ZF2DGVxgM1Ii-nEX0CRdKotqRvK19M5i8mwzjorOnbxL8Ix7zGdtkB-mGHLmTvjU7eJkzk2q_XnRbwsfIfjs2z5UfFCFcoZ5TUSDTp6qjX4hESYx6TWS9OqxgiqiApjPEb2EwaCgYKAcYSARMSFQGOcNnC23uomxEzJUCjzXVCfuqi2Q0173", + refresh_token: "1//0dk2dltGhN4n3CgYIARAAGA0SNwF-L9Irh9E6xb10sQaAjUfbdO1a7hx3bKnKze-_jK8OTHoowHwIynRPPshiFDeb3wtgqAnt9yI", + developer_token: "developer_Token", + }, + }], + }; + + + // destinationResponse = { + // response: { + // error: { + // code: 401, + // message: "Request had invalid authentication credentials. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.", + // errors: [ + // { + // message: "Invalid Credentials", + // domain: "global", + // reason: "authError", + // location: "Authorization", + // locationType: "header", + // }, + // ], + // status: "UNAUTHENTICATED", + // }, + // }, + // status: 401, + // }; + + const { response, status, rudderJobMetadata } = destinationResponse; if (isHttpStatusSuccess(status)) { - // check for Failures - if (response.hasFailures === true) { - if (checkIfFailuresAreRetryable(response)) { - throw new RetryableError( - `Campaign Manager: Retrying during CAMPAIGN_MANAGER response transformation`, - 500, - destinationResponse, - ); - } else { - // abort message - throw new AbortedError( - `Campaign Manager: Aborting during CAMPAIGN_MANAGER response transformation`, - 400, - destinationResponse, - ); + // check for Partial Event failures and Successes + const destPartialStatus = response.status; + + for (const [idx, element] of destPartialStatus.entries()) { + const proxyOutputObj = { + statusCode: 200, + metadata: rudderJobMetadata[idx], + error: "success" + }; + // update status of partial event as per retriable or abortable + if (isEventRetryable(element, proxyOutputObj)) { + proxyOutputObj.statusCode = 500; + } else if (isEventAbortable(element, proxyOutputObj)) { + proxyOutputObj.statusCode = 400; } + responseForPartialEventHandling.push(proxyOutputObj); } return { status, message, destinationResponse, + response: responseForPartialEventHandling }; } + // retry entire batch incase of entire batch delivery failure (eg. oauth error) throw new NetworkError( `Campaign Manager: ${response.error?.message} during CAMPAIGN_MANAGER response transformation`, status, From b2719e71a90c9c2e2bbdedf0a951a3ac0cdfecf0 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 9 Oct 2023 17:05:29 +0530 Subject: [PATCH 07/19] feat: update batching partial events --- src/v0/destinations/campaign_manager/networkHandler.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index 0fea5fefb7..06c90c7b6d 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -134,6 +134,11 @@ const responseHandler = (destinationResponse) => { message: "Floodlight config id: 213123123 was not found.", kind: "dfareporting#conversionError", }, + { + code: "INVALID_ARGUMENT", + message: "gclid: 123 was not found.", + kind: "dfareporting#conversionError", + } ], kind: "dfareporting#conversionStatus", }, From 8eba95b4b0f1aeef258de37f113ebc2192980445 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 9 Oct 2023 17:08:38 +0530 Subject: [PATCH 08/19] feat: update batching partial events --- src/services/destination/nativeIntegration.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 49fb806094..4c83966933 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -182,6 +182,7 @@ export default class NativeIntegrationDestinationService implements IntegrationD destinationType, ) as DeliveryResponse; } catch (err: any) { + // update error block also const metaTO = this.getTags( destinationType, destinationRequest.metadata?.destinationId || 'Non-determininable', From be9aa3345316e430e4fbed180fcf9f67de7731f0 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 9 Oct 2023 17:11:40 +0530 Subject: [PATCH 09/19] feat: update batching partial events --- .../campaign_manager/networkHandler.js | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index 06c90c7b6d..d86e54ff10 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -174,12 +174,12 @@ const responseHandler = (destinationResponse) => { jobId: 10, attemptNum: 0, userId: "", - sourceId: "2Vsge2uWYdrLfG7pZb5Y82eo4lr", - destinationId: "2RHh08uOsXqE9KvCDg3hoaeuK2L", - workspaceId: "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + sourceId: "24242", + destinationId: "24242", + workspaceId: "242424", secret: { - access_token: "ya29.a0AfB_byCptyxhlxHnWGZWiV9MkQNpF_wm6ZF2DGVxgM1Ii-nEX0CRdKotqRvK19M5i8mwzjorOnbxL8Ix7zGdtkB-mGHLmTvjU7eJkzk2q_XnRbwsfIfjs2z5UfFCFcoZ5TUSDTp6qjX4hESYx6TWS9OqxgiqiApjPEb2EwaCgYKAcYSARMSFQGOcNnC23uomxEzJUCjzXVCfuqi2Q0173", - refresh_token: "1//0dk2dltGhN4n3CgYIARAAGA0SNwF-L9Irh9E6xb10sQaAjUfbdO1a7hx3bKnKze-_jK8OTHoowHwIynRPPshiFDeb3wtgqAnt9yI", + access_token: "atoken", + refresh_token: "rtoken", developer_token: "developer_Token", }, }, @@ -187,12 +187,12 @@ const responseHandler = (destinationResponse) => { jobId: 11, attemptNum: 0, userId: "", - sourceId: "2Vsge2uWYdrLfG7pZb5Y82eo4lr", - destinationId: "2RHh08uOsXqE9KvCDg3hoaeuK2L", - workspaceId: "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + sourceId: "2424", + destinationId: "24242", + workspaceId: "24242", secret: { - access_token: "ya29.a0AfB_byCptyxhlxHnWGZWiV9MkQNpF_wm6ZF2DGVxgM1Ii-nEX0CRdKotqRvK19M5i8mwzjorOnbxL8Ix7zGdtkB-mGHLmTvjU7eJkzk2q_XnRbwsfIfjs2z5UfFCFcoZ5TUSDTp6qjX4hESYx6TWS9OqxgiqiApjPEb2EwaCgYKAcYSARMSFQGOcNnC23uomxEzJUCjzXVCfuqi2Q0173", - refresh_token: "1//0dk2dltGhN4n3CgYIARAAGA0SNwF-L9Irh9E6xb10sQaAjUfbdO1a7hx3bKnKze-_jK8OTHoowHwIynRPPshiFDeb3wtgqAnt9yI", + access_token: "atoken", + refresh_token: "rtoken", developer_token: "developer_Token", }, }, @@ -200,12 +200,12 @@ const responseHandler = (destinationResponse) => { jobId: 12, attemptNum: 0, userId: "", - sourceId: "2Vsge2uWYdrLfG7pZb5Y82eo4lr", - destinationId: "2RHh08uOsXqE9KvCDg3hoaeuK2L", - workspaceId: "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + sourceId: "24242", + destinationId: "234234", + workspaceId: "34324", secret: { - access_token: "ya29.a0AfB_byCptyxhlxHnWGZWiV9MkQNpF_wm6ZF2DGVxgM1Ii-nEX0CRdKotqRvK19M5i8mwzjorOnbxL8Ix7zGdtkB-mGHLmTvjU7eJkzk2q_XnRbwsfIfjs2z5UfFCFcoZ5TUSDTp6qjX4hESYx6TWS9OqxgiqiApjPEb2EwaCgYKAcYSARMSFQGOcNnC23uomxEzJUCjzXVCfuqi2Q0173", - refresh_token: "1//0dk2dltGhN4n3CgYIARAAGA0SNwF-L9Irh9E6xb10sQaAjUfbdO1a7hx3bKnKze-_jK8OTHoowHwIynRPPshiFDeb3wtgqAnt9yI", + access_token: "atoken", + refresh_token: "rtoken", developer_token: "developer_Token", }, }], From ddbbfa4953316d1a735eb1328ae1d6f80e335a6f Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Fri, 13 Oct 2023 10:18:26 +0530 Subject: [PATCH 10/19] chore: removed hardcoded response --- .../campaign_manager/networkHandler.js | 259 +++++++++--------- 1 file changed, 133 insertions(+), 126 deletions(-) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index d86e54ff10..9688116c0f 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -7,7 +7,7 @@ const { processAxiosResponse, getDynamicErrorType, } = require('../../../adapters/utils/networkUtils'); -const { NetworkError } = require('../../util/errorTypes'); +const { AbortedError, RetryableError, NetworkError } = require('../../util/errorTypes'); const tags = require('../../util/tags'); // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -92,124 +92,124 @@ function isEventAbortable(element, proxyOutputObj) { const responseHandler = (destinationResponse) => { const message = `[CAMPAIGN_MANAGER Response Handler] - Request Processed Successfully`; const responseForPartialEventHandling = [] - destinationResponse = { - response: { - hasFailures: true, - status: [ - { - conversion: { - floodlightConfigurationId: "213123123", - floodlightActivityId: "456543345245", - timestampMicros: "1668624722555000", - value: 756234234234, - quantity: "3", - ordinal: "1", - limitAdTracking: false, - childDirectedTreatment: false, - gclid: "123", - nonPersonalizedAd: false, - treatmentForUnderage: false, - kind: "dfareporting#conversion", - }, - kind: "dfareporting#conversionStatus", - }, - { - conversion: { - floodlightConfigurationId: "213123123", - floodlightActivityId: "456543345245", - timestampMicros: "1668624722555000", - value: 756234234234, - quantity: "3", - ordinal: "1", - limitAdTracking: false, - childDirectedTreatment: false, - gclid: "123", - nonPersonalizedAd: false, - treatmentForUnderage: false, - kind: "dfareporting#conversion", - }, - errors: [ - { - code: "NOT_FOUND", - message: "Floodlight config id: 213123123 was not found.", - kind: "dfareporting#conversionError", - }, - { - code: "INVALID_ARGUMENT", - message: "gclid: 123 was not found.", - kind: "dfareporting#conversionError", - } - ], - kind: "dfareporting#conversionStatus", - }, - { - conversion: { - floodlightConfigurationId: "213123123", - floodlightActivityId: "456543345245", - timestampMicros: "1668624722555000", - value: 756234234234, - quantity: "3", - ordinal: "1", - limitAdTracking: false, - childDirectedTreatment: false, - gclid: "123", - nonPersonalizedAd: false, - treatmentForUnderage: false, - kind: "dfareporting#conversion", - }, - errors: [ - { - code: "NOT_FOUND", - message: "Floodlight config id: 213123123 was not found.", - kind: "dfareporting#conversionError", - }, - ], - kind: "dfareporting#conversionStatus", - }, - ], - kind: "dfareporting#conversionsBatchInsertResponse", - }, - status: 200, - rudderJobMetadata: [{ - jobId: 10, - attemptNum: 0, - userId: "", - sourceId: "24242", - destinationId: "24242", - workspaceId: "242424", - secret: { - access_token: "atoken", - refresh_token: "rtoken", - developer_token: "developer_Token", - }, - }, - { - jobId: 11, - attemptNum: 0, - userId: "", - sourceId: "2424", - destinationId: "24242", - workspaceId: "24242", - secret: { - access_token: "atoken", - refresh_token: "rtoken", - developer_token: "developer_Token", - }, - }, - { - jobId: 12, - attemptNum: 0, - userId: "", - sourceId: "24242", - destinationId: "234234", - workspaceId: "34324", - secret: { - access_token: "atoken", - refresh_token: "rtoken", - developer_token: "developer_Token", - }, - }], - }; + // destinationResponse = { + // response: { + // hasFailures: true, + // status: [ + // { + // conversion: { + // floodlightConfigurationId: "213123123", + // floodlightActivityId: "456543345245", + // timestampMicros: "1668624722555000", + // value: 756234234234, + // quantity: "3", + // ordinal: "1", + // limitAdTracking: false, + // childDirectedTreatment: false, + // gclid: "123", + // nonPersonalizedAd: false, + // treatmentForUnderage: false, + // kind: "dfareporting#conversion", + // }, + // kind: "dfareporting#conversionStatus", + // }, + // { + // conversion: { + // floodlightConfigurationId: "213123123", + // floodlightActivityId: "456543345245", + // timestampMicros: "1668624722555000", + // value: 756234234234, + // quantity: "3", + // ordinal: "1", + // limitAdTracking: false, + // childDirectedTreatment: false, + // gclid: "123", + // nonPersonalizedAd: false, + // treatmentForUnderage: false, + // kind: "dfareporting#conversion", + // }, + // errors: [ + // { + // code: "NOT_FOUND", + // message: "Floodlight config id: 213123123 was not found.", + // kind: "dfareporting#conversionError", + // }, + // { + // code: "INVALID_ARGUMENT", + // message: "gclid: 123 was not found.", + // kind: "dfareporting#conversionError", + // } + // ], + // kind: "dfareporting#conversionStatus", + // }, + // { + // conversion: { + // floodlightConfigurationId: "213123123", + // floodlightActivityId: "456543345245", + // timestampMicros: "1668624722555000", + // value: 756234234234, + // quantity: "3", + // ordinal: "1", + // limitAdTracking: false, + // childDirectedTreatment: false, + // gclid: "123", + // nonPersonalizedAd: false, + // treatmentForUnderage: false, + // kind: "dfareporting#conversion", + // }, + // errors: [ + // { + // code: "NOT_FOUND", + // message: "Floodlight config id: 213123123 was not found.", + // kind: "dfareporting#conversionError", + // }, + // ], + // kind: "dfareporting#conversionStatus", + // }, + // ], + // kind: "dfareporting#conversionsBatchInsertResponse", + // }, + // status: 200, + // rudderJobMetadata: [{ + // jobId: 10, + // attemptNum: 0, + // userId: "", + // sourceId: "24242", + // destinationId: "24242", + // workspaceId: "242424", + // secret: { + // access_token: "atoken", + // refresh_token: "rtoken", + // developer_token: "developer_Token", + // }, + // }, + // { + // jobId: 11, + // attemptNum: 0, + // userId: "", + // sourceId: "2424", + // destinationId: "24242", + // workspaceId: "24242", + // secret: { + // access_token: "atoken", + // refresh_token: "rtoken", + // developer_token: "developer_Token", + // }, + // }, + // { + // jobId: 12, + // attemptNum: 0, + // userId: "", + // sourceId: "24242", + // destinationId: "234234", + // workspaceId: "34324", + // secret: { + // access_token: "atoken", + // refresh_token: "rtoken", + // developer_token: "developer_Token", + // }, + // }], + // }; // destinationResponse = { @@ -260,16 +260,23 @@ const responseHandler = (destinationResponse) => { }; } - // retry entire batch incase of entire batch delivery failure (eg. oauth error) - throw new NetworkError( - `Campaign Manager: ${response.error?.message} during CAMPAIGN_MANAGER response transformation`, - status, - { - [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), - }, + throw new RetryableError( + `Campaign Manager: Retrying during CAMPAIGN_MANAGER response transformation`, + 500, destinationResponse, getAuthErrCategoryFromStCode(status), ); + + // retry entire batch incase of entire batch delivery failure (eg. oauth error) + // throw new NetworkError( + // `Campaign Manager: ${response.error?.message} during CAMPAIGN_MANAGER response transformation`, + // status, + // { + // [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + // }, + // destinationResponse, + // getAuthErrCategoryFromStCode(status), + // ); }; function networkHandler() { From 18431cb4cffe4e6ade49102875b4c429d93e7b8c Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Tue, 17 Oct 2023 18:16:36 +0530 Subject: [PATCH 11/19] feat: added new error class --- src/services/destination/nativeIntegration.ts | 1 - .../destination/postTransformation.ts | 20 +++++++++- .../campaign_manager/networkHandler.js | 39 ++++++++++--------- src/v0/util/errorTypes/index.js | 2 + .../util/errorTypes/transformerProxyError.js | 31 +++++++++++++++ 5 files changed, 72 insertions(+), 21 deletions(-) create mode 100644 src/v0/util/errorTypes/transformerProxyError.js diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 4c83966933..49fb806094 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -182,7 +182,6 @@ export default class NativeIntegrationDestinationService implements IntegrationD destinationType, ) as DeliveryResponse; } catch (err: any) { - // update error block also const metaTO = this.getTags( destinationType, destinationRequest.metadata?.destinationId || 'Non-determininable', diff --git a/src/services/destination/postTransformation.ts b/src/services/destination/postTransformation.ts index 090e39b059..f791bcbe65 100644 --- a/src/services/destination/postTransformation.ts +++ b/src/services/destination/postTransformation.ts @@ -16,6 +16,16 @@ import ErrorReportingService from '../errorReporting'; import tags from '../../v0/util/tags'; import stats from '../../util/stats'; + +type ErrorResponse = { + status?: number; + message?: string; + destinationResponse?: object; + statTags?: object; + authErrorCategory?: string | undefined; + response?: object | undefined; +}; + export default class DestinationPostTransformationService { public static handleProcessorTransformSucessEvents( event: ProcessorTransformationRequest, @@ -139,7 +149,7 @@ export default class DestinationPostTransformationService { } public static handleDeliveryFailureEvents( - error: NonNullable, + error: ErrorResponse, metaTo: MetaTransferObject, ): DeliveryResponse { const errObj = generateErrorObject(error, metaTo.errorDetails, false); @@ -150,8 +160,14 @@ export default class DestinationPostTransformationService { statTags: errObj.statTags, ...(errObj.authErrorCategory && { authErrorCategory: errObj.authErrorCategory, - }), + }) } as DeliveryResponse; + + // for transformer-proxy to maintain contract + const { response } = error; + if (response) { + resp.response = response; + } ErrorReportingService.reportError(error, metaTo.errorContext, resp); return resp; } diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index 9688116c0f..fc3a181cd9 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -7,7 +7,7 @@ const { processAxiosResponse, getDynamicErrorType, } = require('../../../adapters/utils/networkUtils'); -const { AbortedError, RetryableError, NetworkError } = require('../../util/errorTypes'); +const { TransformerProxyError } = require('../../util/errorTypes'); const tags = require('../../util/tags'); // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -91,7 +91,7 @@ function isEventAbortable(element, proxyOutputObj) { const responseHandler = (destinationResponse) => { const message = `[CAMPAIGN_MANAGER Response Handler] - Request Processed Successfully`; - const responseForPartialEventHandling = [] + const responseWithPartialEvents = [] // destinationResponse = { // response: { // hasFailures: true, @@ -249,34 +249,37 @@ const responseHandler = (destinationResponse) => { } else if (isEventAbortable(element, proxyOutputObj)) { proxyOutputObj.statusCode = 400; } - responseForPartialEventHandling.push(proxyOutputObj); + responseWithPartialEvents.push(proxyOutputObj); } return { status, message, destinationResponse, - response: responseForPartialEventHandling - }; + response: responseWithPartialEvents + } + } + + // in case of failure status, populate response to maintain len(metadata)=len(response) + const errorMessage = response.error?.message || 'error msg failure'; + for (const metadata of rudderJobMetadata) { + responseWithPartialEvents.push({ + statusCode: 500, + metadata, + error: errorMessage + }); } - throw new RetryableError( - `Campaign Manager: Retrying during CAMPAIGN_MANAGER response transformation`, + throw new TransformerProxyError( + `Campaign Manager: Error proxy during CAMPAIGN_MANAGER response transformation`, 500, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, destinationResponse, getAuthErrCategoryFromStCode(status), + responseWithPartialEvents ); - - // retry entire batch incase of entire batch delivery failure (eg. oauth error) - // throw new NetworkError( - // `Campaign Manager: ${response.error?.message} during CAMPAIGN_MANAGER response transformation`, - // status, - // { - // [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), - // }, - // destinationResponse, - // getAuthErrCategoryFromStCode(status), - // ); }; function networkHandler() { diff --git a/src/v0/util/errorTypes/index.js b/src/v0/util/errorTypes/index.js index 01708caa15..460b878058 100644 --- a/src/v0/util/errorTypes/index.js +++ b/src/v0/util/errorTypes/index.js @@ -14,6 +14,7 @@ const UnhandledStatusCodeError = require('./unhandledStatusCodeError'); const UnauthorizedError = require('./unauthorizedError'); const NetworkInstrumentationError = require('./networkInstrumentationError'); const UnsupportedEventError = require('./unsupportedEventError'); +const TransformerProxyError = require('./transformerProxyError'); module.exports = { BaseError, @@ -32,4 +33,5 @@ module.exports = { NetworkInstrumentationError, UnsupportedEventError, RedisError, + TransformerProxyError, }; diff --git a/src/v0/util/errorTypes/transformerProxyError.js b/src/v0/util/errorTypes/transformerProxyError.js new file mode 100644 index 0000000000..42bd55e445 --- /dev/null +++ b/src/v0/util/errorTypes/transformerProxyError.js @@ -0,0 +1,31 @@ +const tags = require('../tags'); + +const errorTypes = Object.values(tags.ERROR_TYPES); +const metaTypes = Object.values(tags.METADATA); +class TransformerProxyError extends Error { + constructor(message, statusCode, statTags, destResponse, authErrorCategory, response) { + const finalStatTags = { + [tags.TAG_NAMES.ERROR_CATEGORY]: tags.ERROR_CATEGORIES.NETWORK, + [tags.TAG_NAMES.ERROR_TYPE]: tags.ERROR_TYPES.ABORTED, + }; + + // Allow specifying only error type and meta tags + if (statTags && typeof statTags === 'object' && !Array.isArray(statTags)) { + if (errorTypes.includes(statTags[tags.TAG_NAMES.ERROR_TYPE])) { + finalStatTags[tags.TAG_NAMES.ERROR_TYPE] = statTags[tags.TAG_NAMES.ERROR_TYPE]; + } + + if (metaTypes.includes(statTags[tags.TAG_NAMES.META])) { + finalStatTags[tags.TAG_NAMES.META] = statTags[tags.TAG_NAMES.META]; + } + } + super(message); + this.status = statusCode; + this.statTags = finalStatTags; + this.destinationResponse = destResponse; + this.authErrorCategory = authErrorCategory; + this.response = response; + } +} + +module.exports = TransformerProxyError; From 7679fcc11b4053e296188735ffbeffe33be6c2b9 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Wed, 18 Oct 2023 19:12:32 +0530 Subject: [PATCH 12/19] chore: develop merge --- src/v0/util/errorTypes/transformerProxyError.js | 9 +++------ .../destinations/campaign_manager/router/data.ts | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/v0/util/errorTypes/transformerProxyError.js b/src/v0/util/errorTypes/transformerProxyError.js index 42bd55e445..42ccdea490 100644 --- a/src/v0/util/errorTypes/transformerProxyError.js +++ b/src/v0/util/errorTypes/transformerProxyError.js @@ -1,8 +1,9 @@ const tags = require('../tags'); +const { BaseError } = require('./base'); const errorTypes = Object.values(tags.ERROR_TYPES); const metaTypes = Object.values(tags.METADATA); -class TransformerProxyError extends Error { +class TransformerProxyError extends BaseError { constructor(message, statusCode, statTags, destResponse, authErrorCategory, response) { const finalStatTags = { [tags.TAG_NAMES.ERROR_CATEGORY]: tags.ERROR_CATEGORIES.NETWORK, @@ -19,11 +20,7 @@ class TransformerProxyError extends Error { finalStatTags[tags.TAG_NAMES.META] = statTags[tags.TAG_NAMES.META]; } } - super(message); - this.status = statusCode; - this.statTags = finalStatTags; - this.destinationResponse = destResponse; - this.authErrorCategory = authErrorCategory; + super(message, statusCode, finalStatTags, destResponse, authErrorCategory); this.response = response; } } diff --git a/test/integrations/destinations/campaign_manager/router/data.ts b/test/integrations/destinations/campaign_manager/router/data.ts index 95372e1925..3f3ba92f65 100644 --- a/test/integrations/destinations/campaign_manager/router/data.ts +++ b/test/integrations/destinations/campaign_manager/router/data.ts @@ -348,7 +348,7 @@ export const data = [ jobId: 1, }, ], - batched: false, + batched: true, statusCode: 200, destination: { Config: { @@ -411,7 +411,7 @@ export const data = [ jobId: 2, }, ], - batched: false, + batched: true, statusCode: 200, destination: { Config: { From a984ad9e34daad89b204a0a06d78f6330b107225 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Thu, 19 Oct 2023 18:25:49 +0530 Subject: [PATCH 13/19] feat: handle metadata as obj --- .../campaign_manager/networkHandler.js | 104 ++++++++++++------ 1 file changed, 72 insertions(+), 32 deletions(-) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index fc3a181cd9..2835f79526 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -7,7 +7,7 @@ const { processAxiosResponse, getDynamicErrorType, } = require('../../../adapters/utils/networkUtils'); -const { TransformerProxyError } = require('../../util/errorTypes'); +const { RetryableError, NetworkError, AbortedError, TransformerProxyError } = require('../../util/errorTypes'); const tags = require('../../util/tags'); // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -233,53 +233,93 @@ const responseHandler = (destinationResponse) => { // }; const { response, status, rudderJobMetadata } = destinationResponse; + + if (Array.isArray(rudderJobMetadata)) { + if (isHttpStatusSuccess(status)) { + // check for Partial Event failures and Successes + const destPartialStatus = response.status; + + for (const [idx, element] of destPartialStatus.entries()) { + const proxyOutputObj = { + statusCode: 200, + metadata: rudderJobMetadata[idx], + error: "success" + }; + // update status of partial event as per retriable or abortable + if (isEventRetryable(element, proxyOutputObj)) { + proxyOutputObj.statusCode = 500; + } else if (isEventAbortable(element, proxyOutputObj)) { + proxyOutputObj.statusCode = 400; + } + responseWithPartialEvents.push(proxyOutputObj); + } + + return { + status, + message, + destinationResponse, + response: responseWithPartialEvents + } + } + + // in case of failure status, populate response to maintain len(metadata)=len(response) + const errorMessage = response.error?.message || 'unknown error format'; + for (const metadata of rudderJobMetadata) { + responseWithPartialEvents.push({ + statusCode: 500, + metadata, + error: errorMessage + }); + } + + throw new TransformerProxyError( + `Campaign Manager: Error proxy during CAMPAIGN_MANAGER response transformation`, + 500, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + getAuthErrCategoryFromStCode(status), + responseWithPartialEvents + ); + } + if (isHttpStatusSuccess(status)) { - // check for Partial Event failures and Successes - const destPartialStatus = response.status; - - for (const [idx, element] of destPartialStatus.entries()) { - const proxyOutputObj = { - statusCode: 200, - metadata: rudderJobMetadata[idx], - error: "success" - }; - // update status of partial event as per retriable or abortable - if (isEventRetryable(element, proxyOutputObj)) { - proxyOutputObj.statusCode = 500; - } else if (isEventAbortable(element, proxyOutputObj)) { - proxyOutputObj.statusCode = 400; + // check for Failures + if (response.hasFailures === true) { + if (checkIfFailuresAreRetryable(response)) { + throw new RetryableError( + `Campaign Manager: Retrying during CAMPAIGN_MANAGER response transformation`, + 500, + destinationResponse, + ); + } else { + // abort message + throw new AbortedError( + `Campaign Manager: Aborting during CAMPAIGN_MANAGER response transformation`, + 400, + destinationResponse, + ); } - responseWithPartialEvents.push(proxyOutputObj); } return { status, message, destinationResponse, - response: responseWithPartialEvents - } + }; } - // in case of failure status, populate response to maintain len(metadata)=len(response) - const errorMessage = response.error?.message || 'error msg failure'; - for (const metadata of rudderJobMetadata) { - responseWithPartialEvents.push({ - statusCode: 500, - metadata, - error: errorMessage - }); - } - - throw new TransformerProxyError( - `Campaign Manager: Error proxy during CAMPAIGN_MANAGER response transformation`, - 500, + throw new NetworkError( + `Campaign Manager: ${response.error?.message} during CAMPAIGN_MANAGER response transformation 3`, + status, { [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), }, destinationResponse, getAuthErrCategoryFromStCode(status), - responseWithPartialEvents ); + }; function networkHandler() { From c09d83df3c64dda30e0b012be07cae0e1d6acd92 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Thu, 19 Oct 2023 18:40:59 +0530 Subject: [PATCH 14/19] feat: handle metadata as obj --- src/v0/destinations/campaign_manager/networkHandler.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index 2835f79526..6b2300fce1 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -9,6 +9,7 @@ const { } = require('../../../adapters/utils/networkUtils'); const { RetryableError, NetworkError, AbortedError, TransformerProxyError } = require('../../util/errorTypes'); const tags = require('../../util/tags'); +const logger = require('../../../logger'); // eslint-disable-next-line @typescript-eslint/no-unused-vars function checkIfFailuresAreRetryable(response, proxyOutputObj) { @@ -235,6 +236,12 @@ const responseHandler = (destinationResponse) => { const { response, status, rudderJobMetadata } = destinationResponse; if (Array.isArray(rudderJobMetadata)) { + try { + logger.info('length of medatada: ', rudderJobMetadata.length); + logger.info('number of events: ', response.status.length); + } catch (error) { + logger.error(error); + } if (isHttpStatusSuccess(status)) { // check for Partial Event failures and Successes const destPartialStatus = response.status; From 9b7416f36764376227404b6147b51c7d10378a47 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Fri, 20 Oct 2023 01:36:51 +0530 Subject: [PATCH 15/19] chore: removed commented code --- .../campaign_manager/networkHandler.js | 157 +----------------- 1 file changed, 5 insertions(+), 152 deletions(-) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index 6b2300fce1..60afe0f6f8 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -9,7 +9,6 @@ const { } = require('../../../adapters/utils/networkUtils'); const { RetryableError, NetworkError, AbortedError, TransformerProxyError } = require('../../util/errorTypes'); const tags = require('../../util/tags'); -const logger = require('../../../logger'); // eslint-disable-next-line @typescript-eslint/no-unused-vars function checkIfFailuresAreRetryable(response, proxyOutputObj) { @@ -92,156 +91,10 @@ function isEventAbortable(element, proxyOutputObj) { const responseHandler = (destinationResponse) => { const message = `[CAMPAIGN_MANAGER Response Handler] - Request Processed Successfully`; - const responseWithPartialEvents = [] - // destinationResponse = { - // response: { - // hasFailures: true, - // status: [ - // { - // conversion: { - // floodlightConfigurationId: "213123123", - // floodlightActivityId: "456543345245", - // timestampMicros: "1668624722555000", - // value: 756234234234, - // quantity: "3", - // ordinal: "1", - // limitAdTracking: false, - // childDirectedTreatment: false, - // gclid: "123", - // nonPersonalizedAd: false, - // treatmentForUnderage: false, - // kind: "dfareporting#conversion", - // }, - // kind: "dfareporting#conversionStatus", - // }, - // { - // conversion: { - // floodlightConfigurationId: "213123123", - // floodlightActivityId: "456543345245", - // timestampMicros: "1668624722555000", - // value: 756234234234, - // quantity: "3", - // ordinal: "1", - // limitAdTracking: false, - // childDirectedTreatment: false, - // gclid: "123", - // nonPersonalizedAd: false, - // treatmentForUnderage: false, - // kind: "dfareporting#conversion", - // }, - // errors: [ - // { - // code: "NOT_FOUND", - // message: "Floodlight config id: 213123123 was not found.", - // kind: "dfareporting#conversionError", - // }, - // { - // code: "INVALID_ARGUMENT", - // message: "gclid: 123 was not found.", - // kind: "dfareporting#conversionError", - // } - // ], - // kind: "dfareporting#conversionStatus", - // }, - // { - // conversion: { - // floodlightConfigurationId: "213123123", - // floodlightActivityId: "456543345245", - // timestampMicros: "1668624722555000", - // value: 756234234234, - // quantity: "3", - // ordinal: "1", - // limitAdTracking: false, - // childDirectedTreatment: false, - // gclid: "123", - // nonPersonalizedAd: false, - // treatmentForUnderage: false, - // kind: "dfareporting#conversion", - // }, - // errors: [ - // { - // code: "NOT_FOUND", - // message: "Floodlight config id: 213123123 was not found.", - // kind: "dfareporting#conversionError", - // }, - // ], - // kind: "dfareporting#conversionStatus", - // }, - // ], - // kind: "dfareporting#conversionsBatchInsertResponse", - // }, - // status: 200, - // rudderJobMetadata: [{ - // jobId: 10, - // attemptNum: 0, - // userId: "", - // sourceId: "24242", - // destinationId: "24242", - // workspaceId: "242424", - // secret: { - // access_token: "atoken", - // refresh_token: "rtoken", - // developer_token: "developer_Token", - // }, - // }, - // { - // jobId: 11, - // attemptNum: 0, - // userId: "", - // sourceId: "2424", - // destinationId: "24242", - // workspaceId: "24242", - // secret: { - // access_token: "atoken", - // refresh_token: "rtoken", - // developer_token: "developer_Token", - // }, - // }, - // { - // jobId: 12, - // attemptNum: 0, - // userId: "", - // sourceId: "24242", - // destinationId: "234234", - // workspaceId: "34324", - // secret: { - // access_token: "atoken", - // refresh_token: "rtoken", - // developer_token: "developer_Token", - // }, - // }], - // }; - - - // destinationResponse = { - // response: { - // error: { - // code: 401, - // message: "Request had invalid authentication credentials. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.", - // errors: [ - // { - // message: "Invalid Credentials", - // domain: "global", - // reason: "authError", - // location: "Authorization", - // locationType: "header", - // }, - // ], - // status: "UNAUTHENTICATED", - // }, - // }, - // status: 401, - // }; - + const responseWithIndividualEvents = []; const { response, status, rudderJobMetadata } = destinationResponse; if (Array.isArray(rudderJobMetadata)) { - try { - logger.info('length of medatada: ', rudderJobMetadata.length); - logger.info('number of events: ', response.status.length); - } catch (error) { - logger.error(error); - } if (isHttpStatusSuccess(status)) { // check for Partial Event failures and Successes const destPartialStatus = response.status; @@ -258,21 +111,21 @@ const responseHandler = (destinationResponse) => { } else if (isEventAbortable(element, proxyOutputObj)) { proxyOutputObj.statusCode = 400; } - responseWithPartialEvents.push(proxyOutputObj); + responseWithIndividualEvents.push(proxyOutputObj); } return { status, message, destinationResponse, - response: responseWithPartialEvents + response: responseWithIndividualEvents } } // in case of failure status, populate response to maintain len(metadata)=len(response) const errorMessage = response.error?.message || 'unknown error format'; for (const metadata of rudderJobMetadata) { - responseWithPartialEvents.push({ + responseWithIndividualEvents.push({ statusCode: 500, metadata, error: errorMessage @@ -287,7 +140,7 @@ const responseHandler = (destinationResponse) => { }, destinationResponse, getAuthErrCategoryFromStCode(status), - responseWithPartialEvents + responseWithIndividualEvents ); } From 1b7ba32fff39e1b3e27a39e9c5766c964818635d Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Tue, 21 Nov 2023 16:20:12 +0530 Subject: [PATCH 16/19] fix: merged develop --- src/services/destination/postTransformation.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/destination/postTransformation.ts b/src/services/destination/postTransformation.ts index ffb7d5734e..1e99961045 100644 --- a/src/services/destination/postTransformation.ts +++ b/src/services/destination/postTransformation.ts @@ -25,7 +25,7 @@ type ErrorResponse = { response?: object | undefined; }; -export default class DestinationPostTransformationService { +export class DestinationPostTransformationService { public static handleProcessorTransformSucessEvents( event: ProcessorTransformationRequest, transformedPayloads: ProcessorTransformationOutput | ProcessorTransformationOutput[], @@ -159,7 +159,7 @@ export default class DestinationPostTransformationService { statTags: errObj.statTags, ...(errObj.authErrorCategory && { authErrorCategory: errObj.authErrorCategory, - }) + }), } as DeliveryResponse; // for transformer-proxy to maintain contract From 756ca99f67d0627ac1baa1111075fab5ff8ef178 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Thu, 30 Nov 2023 11:37:01 +0530 Subject: [PATCH 17/19] fix: merged develop --- src/v0/destinations/am/transform.js | 54 ++++----- .../campaign_manager/networkHandler.js | 103 +----------------- src/v0/util/errorTypes/index.js | 2 - .../util/errorTypes/transformerProxyError.js | 28 ----- .../campaign_manager/dataDelivery/data.ts | 8 +- 5 files changed, 36 insertions(+), 159 deletions(-) delete mode 100644 src/v0/util/errorTypes/transformerProxyError.js diff --git a/src/v0/destinations/am/transform.js b/src/v0/destinations/am/transform.js index 3be2b0e985..05a130d6e0 100644 --- a/src/v0/destinations/am/transform.js +++ b/src/v0/destinations/am/transform.js @@ -591,8 +591,12 @@ const processSingleMessage = (message, destination) => { const { name, event, properties } = message; const messageType = message.type.toLowerCase(); const CATEGORY_KEY = 'properties.category'; - const { useUserDefinedPageEventName, userProvidedPageEventString, - useUserDefinedScreenEventName, userProvidedScreenEventString } = destination.Config; + const { + useUserDefinedPageEventName, + userProvidedPageEventString, + useUserDefinedScreenEventName, + userProvidedScreenEventString, + } = destination.Config; switch (messageType) { case EventType.IDENTIFY: payloadObjectName = 'events'; // identify same as events @@ -602,17 +606,17 @@ const processSingleMessage = (message, destination) => { case EventType.PAGE: if (useUserDefinedPageEventName) { const getMessagePath = userProvidedPageEventString - .substring( - userProvidedPageEventString.indexOf('{') + 2, - userProvidedPageEventString.indexOf('}'), - ) - .trim(); + .substring( + userProvidedPageEventString.indexOf('{') + 2, + userProvidedPageEventString.indexOf('}'), + ) + .trim(); evType = - userProvidedPageEventString.trim() === '' - ? name - : userProvidedPageEventString - .trim() - .replaceAll(/{{([^{}]+)}}/g, get(message, getMessagePath)); + userProvidedPageEventString.trim() === '' + ? name + : userProvidedPageEventString + .trim() + .replaceAll(/{{([^{}]+)}}/g, get(message, getMessagePath)); } else { evType = `Viewed ${name || get(message, CATEGORY_KEY) || ''} Page`; } @@ -625,25 +629,25 @@ const processSingleMessage = (message, destination) => { case EventType.SCREEN: { const { eventType, updatedProperties } = getScreenevTypeAndUpdatedProperties( - message, - CATEGORY_KEY, + message, + CATEGORY_KEY, ); let customScreenEv = ''; if (useUserDefinedScreenEventName) { const getMessagePath = userProvidedScreenEventString - .substring( - userProvidedScreenEventString.indexOf('{') + 2, - userProvidedScreenEventString.indexOf('}'), - ) - .trim(); + .substring( + userProvidedScreenEventString.indexOf('{') + 2, + userProvidedScreenEventString.indexOf('}'), + ) + .trim(); customScreenEv = - userProvidedScreenEventString.trim() === '' - ? name - : userProvidedScreenEventString - .trim() - .replaceAll(/{{([^{}]+)}}/g, get(message, getMessagePath)); + userProvidedScreenEventString.trim() === '' + ? name + : userProvidedScreenEventString + .trim() + .replaceAll(/{{([^{}]+)}}/g, get(message, getMessagePath)); } - evType = customScreenEv || eventType; + evType = useUserDefinedScreenEventName ? customScreenEv : eventType; message.properties = updatedProperties; category = ConfigCategory.SCREEN; } diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index 8bd51e20e6..691e7f19c5 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -1,6 +1,5 @@ -const { AbortedError, RetryableError, NetworkError } = require('@rudderstack/integrations-lib'); -/* eslint-disable no-param-reassign */ /* eslint-disable no-restricted-syntax */ +const { AbortedError, RetryableError, NetworkError } = require('@rudderstack/integrations-lib'); const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); const { isHttpStatusSuccess, getAuthErrCategoryFromStCode } = require('../../util/index'); @@ -8,7 +7,6 @@ const { processAxiosResponse, getDynamicErrorType, } = require('../../../adapters/utils/networkUtils'); -const { TransformerProxyError } = require('../../util/errorTypes'); const tags = require('../../util/tags'); // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -29,7 +27,7 @@ function checkIfFailuresAreRetryable(response, proxyOutputObj) { "kind": string }] */ for (const st of status) { - for(const err of st.errors) { + for (const err of st.errors) { // if code is any of these, event is not retryable if ( err.code === 'PERMISSION_DENIED' || @@ -47,103 +45,9 @@ function checkIfFailuresAreRetryable(response, proxyOutputObj) { } } -function isEventRetryable(element, proxyOutputObj) { - let flag = false; - let errorMsg = ""; - // success event - if (!element.errors) { - return flag; - } - for(const err of element.errors) { - errorMsg += `${err.message}, `; - if (err.code === 'INTERNAL') { - flag = true; - } - } - if (errorMsg) { - proxyOutputObj.error = errorMsg; - } - return flag; -} - -function isEventAbortable(element, proxyOutputObj) { - let flag = false; - let errorMsg = ""; - // success event - if (!element.errors) { - return flag; - } - for(const err of element.errors) { - errorMsg += `${err.message}, `; - // if code is any of these, event is not retryable - if ( - err.code === 'PERMISSION_DENIED' || - err.code === 'INVALID_ARGUMENT' || - err.code === 'NOT_FOUND' - ) { - flag = true; - } - } - if (errorMsg) { - proxyOutputObj.error = errorMsg; - } - return flag; -} - const responseHandler = (destinationResponse) => { const message = `[CAMPAIGN_MANAGER Response Handler] - Request Processed Successfully`; - const responseWithIndividualEvents = []; - const { response, status, rudderJobMetadata } = destinationResponse; - - if (Array.isArray(rudderJobMetadata)) { - if (isHttpStatusSuccess(status)) { - // check for Partial Event failures and Successes - const destPartialStatus = response.status; - - for (const [idx, element] of destPartialStatus.entries()) { - const proxyOutputObj = { - statusCode: 200, - metadata: rudderJobMetadata[idx], - error: "success" - }; - // update status of partial event as per retriable or abortable - if (isEventRetryable(element, proxyOutputObj)) { - proxyOutputObj.statusCode = 500; - } else if (isEventAbortable(element, proxyOutputObj)) { - proxyOutputObj.statusCode = 400; - } - responseWithIndividualEvents.push(proxyOutputObj); - } - - return { - status, - message, - destinationResponse, - response: responseWithIndividualEvents - } - } - - // in case of failure status, populate response to maintain len(metadata)=len(response) - const errorMessage = response.error?.message || 'unknown error format'; - for (const metadata of rudderJobMetadata) { - responseWithIndividualEvents.push({ - statusCode: 500, - metadata, - error: errorMessage - }); - } - - throw new TransformerProxyError( - `Campaign Manager: Error proxy during CAMPAIGN_MANAGER response transformation`, - 500, - { - [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), - }, - destinationResponse, - getAuthErrCategoryFromStCode(status), - responseWithIndividualEvents - ); - } + const { response, status } = destinationResponse; if (isHttpStatusSuccess(status)) { // check for Failures @@ -180,7 +84,6 @@ const responseHandler = (destinationResponse) => { destinationResponse, getAuthErrCategoryFromStCode(status), ); - }; function networkHandler() { diff --git a/src/v0/util/errorTypes/index.js b/src/v0/util/errorTypes/index.js index fa39801b78..80268c3e77 100644 --- a/src/v0/util/errorTypes/index.js +++ b/src/v0/util/errorTypes/index.js @@ -1,7 +1,5 @@ -const TransformerProxyError = require('./transformerProxyError'); const FilteredEventsError = require('./filteredEventsError'); module.exports = { - TransformerProxyError, FilteredEventsError, }; diff --git a/src/v0/util/errorTypes/transformerProxyError.js b/src/v0/util/errorTypes/transformerProxyError.js deleted file mode 100644 index 42ccdea490..0000000000 --- a/src/v0/util/errorTypes/transformerProxyError.js +++ /dev/null @@ -1,28 +0,0 @@ -const tags = require('../tags'); -const { BaseError } = require('./base'); - -const errorTypes = Object.values(tags.ERROR_TYPES); -const metaTypes = Object.values(tags.METADATA); -class TransformerProxyError extends BaseError { - constructor(message, statusCode, statTags, destResponse, authErrorCategory, response) { - const finalStatTags = { - [tags.TAG_NAMES.ERROR_CATEGORY]: tags.ERROR_CATEGORIES.NETWORK, - [tags.TAG_NAMES.ERROR_TYPE]: tags.ERROR_TYPES.ABORTED, - }; - - // Allow specifying only error type and meta tags - if (statTags && typeof statTags === 'object' && !Array.isArray(statTags)) { - if (errorTypes.includes(statTags[tags.TAG_NAMES.ERROR_TYPE])) { - finalStatTags[tags.TAG_NAMES.ERROR_TYPE] = statTags[tags.TAG_NAMES.ERROR_TYPE]; - } - - if (metaTypes.includes(statTags[tags.TAG_NAMES.META])) { - finalStatTags[tags.TAG_NAMES.META] = statTags[tags.TAG_NAMES.META]; - } - } - super(message, statusCode, finalStatTags, destResponse, authErrorCategory); - this.response = response; - } -} - -module.exports = TransformerProxyError; diff --git a/test/integrations/destinations/campaign_manager/dataDelivery/data.ts b/test/integrations/destinations/campaign_manager/dataDelivery/data.ts index f2a7654e3c..b7b369941b 100644 --- a/test/integrations/destinations/campaign_manager/dataDelivery/data.ts +++ b/test/integrations/destinations/campaign_manager/dataDelivery/data.ts @@ -135,14 +135,14 @@ export const data = [ }, output: { response: { - status: 500, + status: 400, body: { output: { - status: 500, - message: 'Campaign Manager: Retrying during CAMPAIGN_MANAGER response transformation', + status: 400, + message: 'Campaign Manager: Aborting during CAMPAIGN_MANAGER response transformation', statTags: { errorCategory: 'network', - errorType: 'retryable', + errorType: 'aborted', destType: 'CAMPAIGN_MANAGER', module: 'destination', implementation: 'native', From 68bcea75d774895c296dcfa7dce1092d83cfcdbd Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Thu, 30 Nov 2023 12:25:13 +0530 Subject: [PATCH 18/19] chore: added tests for batching --- .../campaign_manager/dataDelivery/data.ts | 2 +- .../campaign_manager/router/data.ts | 815 +++++++++++++++++- 2 files changed, 805 insertions(+), 12 deletions(-) diff --git a/test/integrations/destinations/campaign_manager/dataDelivery/data.ts b/test/integrations/destinations/campaign_manager/dataDelivery/data.ts index b7b369941b..601ad56401 100644 --- a/test/integrations/destinations/campaign_manager/dataDelivery/data.ts +++ b/test/integrations/destinations/campaign_manager/dataDelivery/data.ts @@ -1,7 +1,7 @@ export const data = [ { name: 'campaign_manager', - description: 'Sucess insert request', + description: 'Sucess insert request V0', feature: 'dataDelivery', module: 'destination', version: 'v0', diff --git a/test/integrations/destinations/campaign_manager/router/data.ts b/test/integrations/destinations/campaign_manager/router/data.ts index 3f3ba92f65..2ab1813cf8 100644 --- a/test/integrations/destinations/campaign_manager/router/data.ts +++ b/test/integrations/destinations/campaign_manager/router/data.ts @@ -1,7 +1,7 @@ export const data = [ { name: 'campaign_manager', - description: 'Test 0', + description: 'Batch Different Type Requests', feature: 'router', module: 'destination', version: 'v0', @@ -77,7 +77,7 @@ export const data = [ properties: { profileId: 437689, floodlightConfigurationId: '213123123', - ordinal: 'string', + ordinal: '1', quantity: '455678', floodlightActivityId: '456543345245', value: 7, @@ -167,7 +167,7 @@ export const data = [ properties: { profileId: 437689, floodlightConfigurationId: '213123123', - ordinal: 'string', + ordinal: '2', floodlightActivityId: '456543345245', quantity: '455678', value: 7, @@ -256,16 +256,11 @@ export const data = [ properties: { profileId: 437689, floodlightConfigurationId: '213123123', - ordinal: 'string', + ordinal: '3', floodlightActivityId: '456543345245', - mobileDeviceId: 'string', value: 7, encryptedUserIdCandidates: ['dfghjbnm'], - gclid: 'string', - matchId: 'string', - dclid: 'string', quantity: '455678', - impressionId: 'string', limitAdTracking: true, childDirectedTreatment: true, encryptionInfo: { @@ -322,7 +317,7 @@ export const data = [ treatmentForUnderage: false, timestampMicros: '1668624722903000', floodlightConfigurationId: '213123123', - ordinal: 'string', + ordinal: '1', quantity: '455678', floodlightActivityId: '456543345245', value: 7, @@ -387,7 +382,7 @@ export const data = [ treatmentForUnderage: false, timestampMicros: '1668624722903000', floodlightConfigurationId: '213123123', - ordinal: 'string', + ordinal: '2', quantity: '455678', floodlightActivityId: '456543345245', value: 7, @@ -461,4 +456,802 @@ export const data = [ }, }, }, + { + name: 'campaign_manager', + description: 'Batch Sucessful BatchInsert Request', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 4, + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + message: { + channel: 'web', + event: 'Promotion Clicked', + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: 437689, + matchId: '123', + floodlightConfigurationId: '213123123', + quantity: '455678', + ordinal: '1', + floodlightActivityId: '456543345245', + value: 7, + limitAdTracking: true, + childDirectedTreatment: true, + requestType: 'batchinsert', + }, + type: 'track', + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2022-11-17T00:22:02.903+05:30', + }, + }, + { + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 5, + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + message: { + channel: 'web', + event: 'Promotion Clicked', + type: 'track', + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: 437689, + floodlightConfigurationId: '213123123', + ordinal: '1', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 7, + matchId: '111', + limitAdTracking: true, + childDirectedTreatment: true, + requestType: 'batchinsert', + }, + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2022-11-17T00:22:02.903+05:30', + }, + }, + { + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 6, + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + message: { + channel: 'web', + event: 'Promotion Clicked', + type: 'track', + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: 437689, + floodlightConfigurationId: '213123123', + floodlightActivityId: '456543345245', + value: 7, + gclid: '123', + ordinal: '1', + quantity: '455678', + limitAdTracking: true, + childDirectedTreatment: true, + requestType: 'batchinsert', + }, + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2022-11-17T00:22:02.903+05:30', + }, + }, + ], + destType: 'campaign_manager', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/437689/conversions/batchinsert', + headers: { + Authorization: 'Bearer dummyApiToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + kind: 'dfareporting#conversionsBatchInsertRequest', + conversions: [ + { + floodlightConfigurationId: '213123123', + ordinal: '1', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 7, + matchId: '123', + limitAdTracking: true, + childDirectedTreatment: true, + nonPersonalizedAd: false, + treatmentForUnderage: false, + }, + { + floodlightConfigurationId: '213123123', + ordinal: '1', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 7, + matchId: '111', + limitAdTracking: true, + childDirectedTreatment: true, + nonPersonalizedAd: false, + treatmentForUnderage: false, + }, + { + floodlightConfigurationId: '213123123', + ordinal: '1', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 7, + gclid: '123', + limitAdTracking: true, + childDirectedTreatment: true, + nonPersonalizedAd: false, + treatmentForUnderage: false, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 4, + }, + { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 5, + }, + { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 6, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + }, + ], + }, + }, + }, + }, + { + name: 'campaign_manager', + description: 'Batch Sucessful BatchInsert and BatchUpdate Request', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 4, + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + message: { + channel: 'web', + event: 'Promotion Clicked', + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: 437689, + matchId: '123', + floodlightConfigurationId: '213123123', + quantity: '455678', + ordinal: '1', + floodlightActivityId: '456543345245', + value: 7, + limitAdTracking: true, + childDirectedTreatment: true, + requestType: 'batchupdate', + }, + type: 'track', + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2022-11-17T00:22:02.903+05:30', + }, + }, + { + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 5, + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + message: { + channel: 'web', + event: 'Promotion Clicked', + type: 'track', + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: 437689, + floodlightConfigurationId: '213123123', + ordinal: '1', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 7, + matchId: '111', + limitAdTracking: true, + childDirectedTreatment: true, + requestType: 'batchupdate', + }, + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2022-11-17T00:22:02.903+05:30', + }, + }, + { + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 6, + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + message: { + channel: 'web', + event: 'Promotion Clicked', + type: 'track', + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: 437689, + floodlightConfigurationId: '213123123', + floodlightActivityId: '456543345245', + value: 7, + gclid: '123', + ordinal: '1', + quantity: '455678', + limitAdTracking: true, + childDirectedTreatment: true, + requestType: 'batchinsert', + }, + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2022-11-17T00:22:02.903+05:30', + }, + }, + ], + destType: 'campaign_manager', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/437689/conversions/batchupdate', + headers: { + Authorization: 'Bearer dummyApiToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + kind: 'dfareporting#conversionsBatchUpdateRequest', + conversions: [ + { + floodlightConfigurationId: '213123123', + ordinal: '1', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 7, + matchId: '123', + nonPersonalizedAd: false, + treatmentForUnderage: false, + }, + { + floodlightConfigurationId: '213123123', + ordinal: '1', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 7, + matchId: '111', + nonPersonalizedAd: false, + treatmentForUnderage: false, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 4, + }, + { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 5, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/437689/conversions/batchinsert', + headers: { + Authorization: 'Bearer dummyApiToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + kind: 'dfareporting#conversionsBatchInsertRequest', + conversions: [ + { + floodlightConfigurationId: '213123123', + ordinal: '1', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 7, + gclid: '123', + limitAdTracking: true, + childDirectedTreatment: true, + nonPersonalizedAd: false, + treatmentForUnderage: false, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 6, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + }, + ], + }, + }, + }, + }, + { + name: 'campaign_manager', + description: 'Entire Batch has data instrumentation', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 4, + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + message: { + channel: 'web', + event: 'Promotion Clicked', + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: 437689, + floodlightConfigurationId: '213123123', + quantity: '455678', + ordinal: '1', + floodlightActivityId: '456543345245', + value: 7, + limitAdTracking: true, + childDirectedTreatment: true, + requestType: 'batchupdate', + }, + type: 'track', + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2022-11-17T00:22:02.903+05:30', + }, + }, + { + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 5, + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + message: { + channel: 'web', + event: 'Promotion Clicked', + type: 'track', + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: 437689, + floodlightConfigurationId: '213123123', + ordinal: '1', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 7, + limitAdTracking: true, + childDirectedTreatment: true, + requestType: 'batchupdate', + }, + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2022-11-17T00:22:02.903+05:30', + }, + }, + { + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 6, + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + message: { + channel: 'web', + event: 'Promotion Clicked', + type: 'track', + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: 437689, + floodlightConfigurationId: '213123123', + floodlightActivityId: '456543345245', + value: 7, + + ordinal: '1', + quantity: '455678', + limitAdTracking: true, + childDirectedTreatment: true, + requestType: 'batchinsert', + }, + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2022-11-17T00:22:02.903+05:30', + }, + }, + ], + destType: 'campaign_manager', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + metadata: [ + { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 4, + }, + ], + batched: false, + statusCode: 400, + error: + '[CAMPAIGN MANAGER (DCM)]: Atleast one of encryptedUserId,encryptedUserIdCandidates, matchId, mobileDeviceId, gclid, dclid, impressionId.', + statTags: { + errorCategory: 'dataValidation', + errorType: 'instrumentation', + destType: 'CAMPAIGN_MANAGER', + module: 'destination', + implementation: 'native', + feature: 'router', + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + }, + { + metadata: [ + { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 5, + }, + ], + batched: false, + statusCode: 400, + error: + '[CAMPAIGN MANAGER (DCM)]: Atleast one of encryptedUserId,encryptedUserIdCandidates, matchId, mobileDeviceId, gclid, dclid, impressionId.', + statTags: { + errorCategory: 'dataValidation', + errorType: 'instrumentation', + destType: 'CAMPAIGN_MANAGER', + module: 'destination', + implementation: 'native', + feature: 'router', + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + }, + { + metadata: [ + { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 6, + }, + ], + batched: false, + statusCode: 400, + error: + '[CAMPAIGN MANAGER (DCM)]: Atleast one of encryptedUserId,encryptedUserIdCandidates, matchId, mobileDeviceId, gclid, dclid, impressionId.', + statTags: { + errorCategory: 'dataValidation', + errorType: 'instrumentation', + destType: 'CAMPAIGN_MANAGER', + module: 'destination', + implementation: 'native', + feature: 'router', + }, + destination: { + Config: { + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + }, + ], + }, + }, + }, + }, ]; From db5991825e36784ea5255b4d34d2cfc5c0f45f1c Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Thu, 30 Nov 2023 12:43:37 +0530 Subject: [PATCH 19/19] chore: added tests for batching --- src/v0/destinations/campaign_manager/networkHandler.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index 691e7f19c5..63efff5b50 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -9,8 +9,7 @@ const { } = require('../../../adapters/utils/networkUtils'); const tags = require('../../util/tags'); -// eslint-disable-next-line @typescript-eslint/no-unused-vars -function checkIfFailuresAreRetryable(response, proxyOutputObj) { +function checkIfFailuresAreRetryable(response) { const { status } = response; try { if (Array.isArray(status)) {