From f44a10533fe49888f8254e29795c3b9368201788 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 25 Sep 2023 13:49:17 +0530 Subject: [PATCH 1/5] fix: timestamp microseconds input cm360 --- .../data/CampaignManagerTrackConfig.json | 6 +- .../campaign_manager/transform.js | 51 ++- test/__tests__/data/campaign_manager.json | 351 +++++++++++++++++- .../data/campaign_manager_router_output.json | 4 +- 4 files changed, 365 insertions(+), 47 deletions(-) diff --git a/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json b/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json index e547bb0f35..f36ba8da8a 100644 --- a/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json +++ b/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json @@ -12,11 +12,7 @@ { "destKey": "timestampMicros", "sourceKeys": "timestamp", - "sourceFromGenericMap": true, - "required": true, - "metadata": { - "type": "microSecondTimestamp" - } + "sourceFromGenericMap": true }, { "destKey": "floodlightActivityId", diff --git a/src/v0/destinations/campaign_manager/transform.js b/src/v0/destinations/campaign_manager/transform.js index 99e5fe9c57..723980f0e4 100644 --- a/src/v0/destinations/campaign_manager/transform.js +++ b/src/v0/destinations/campaign_manager/transform.js @@ -1,5 +1,4 @@ const { EventType } = require('../../../constants'); - const { constructPayload, defaultRequestConfig, @@ -72,7 +71,17 @@ function processTrack(message, metadata, destination) { delete requestJson.childDirectedTreatment; delete requestJson.limitAdTracking; } - requestJson.timestampMicros = requestJson.timestampMicros.toString(); + + const date = new Date(requestJson.timestampMicros); + let unixTimestamp = date.getTime(); + // Date, moment both are not able to distinguish input if it is second,millisecond or microsecond unix timestamp + // Using count of digits to distinguish between the 3, 9999999999999 (13 digits) means Nov 20 2286 + if (unixTimestamp.toString().length === 13) { // milliseconds + unixTimestamp *= 1000; + } else if (unixTimestamp.toString().length === 10) { // seconds + unixTimestamp *= 1000000; + } + requestJson.timestampMicros = unixTimestamp.toString(); const encryptionInfo = {}; // prepare encrptionInfo if encryptedUserId or encryptedUserIdCandidates is given @@ -138,35 +147,17 @@ function postValidateRequest(response) { ); } - let count = 0; - - if (response.body.JSON.conversions[0].gclid) { - count += 1; - } - - if (response.body.JSON.conversions[0].dclid) { - count += 1; - } - - if (response.body.JSON.conversions[0].encryptedUserId) { - count += 1; - } - - if (response.body.JSON.conversions[0].encryptedUserIdCandidates) { - count += 1; - } - - if (response.body.JSON.conversions[0].mobileDeviceId) { - count += 1; - } - - if (response.body.JSON.conversions[0].impressionId) { - count += 1; - } - - if (count !== 1) { + if ( + !response.body.JSON.conversions[0].gclid && + !response.body.JSON.conversions[0].matchId && + !response.body.JSON.conversions[0].dclid && + !response.body.JSON.conversions[0].encryptedUserId && + !response.body.JSON.conversions[0].encryptedUserIdCandidates && + !response.body.JSON.conversions[0].mobileDeviceId && + !response.body.JSON.conversions[0].impressionId + ) { throw new InstrumentationError( - '[CAMPAIGN MANAGER (DCM)]: For CM360 we need one of encryptedUserId,encryptedUserIdCandidates, matchId, mobileDeviceId, gclid, dclid, impressionId.', + '[CAMPAIGN MANAGER (DCM)]: Atleast one of encryptedUserId,encryptedUserIdCandidates, matchId, mobileDeviceId, gclid, dclid, impressionId.', ); } } diff --git a/test/__tests__/data/campaign_manager.json b/test/__tests__/data/campaign_manager.json index ac5b363f64..6139ddbeb3 100644 --- a/test/__tests__/data/campaign_manager.json +++ b/test/__tests__/data/campaign_manager.json @@ -46,8 +46,6 @@ "density": 2 } }, - "event": "Promotion Clicked", - "type": "track", "originalTimestamp": "2022-11-17T00:22:02.903+05:30", "properties": { "profileId": "34245", @@ -112,7 +110,7 @@ { "floodlightConfigurationId": "213123123", "ordinal": "string", - "timestampMicros": "1668624722000000", + "timestampMicros": "1668624722903000", "floodlightActivityId": "456543345245", "quantity": "455678", "value": 756, @@ -178,9 +176,7 @@ "density": 2 } }, - "event": "Promotion Clicked", - "type": "track", - "originalTimestamp": "2022-11-17T00:22:02.903+05:30", + "originalTimestamp": "2020-01-24T11:59:02.402+05:30", "properties": { "profileId": "34245", "floodlightConfigurationId": "213123123", @@ -238,7 +234,7 @@ { "floodlightConfigurationId": "213123123", "ordinal": "string", - "timestampMicros": "1668624722000000", + "timestampMicros": "1579847342402000", "floodlightActivityId": "456543345245", "quantity": "455678", "value": 756, @@ -256,7 +252,7 @@ } }, { - "description": "Track - batch insert Call", + "description": "Track - batch insert Call failure", "input": { "message": { "channel": "web", @@ -302,8 +298,6 @@ "density": 2 } }, - "event": "Promotion Clicked", - "type": "track", "originalTimestamp": "2022-11-17T00:22:02.903+05:30", "properties": { "profileId": "34245", @@ -356,5 +350,342 @@ "scope": "exception" } } + }, + { + "description": "Track - batch update Call timestamp as unix", + "input": { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "device": { + "id": "0572f78fa49c648e", + "name": "generic_x86_arm", + "type": "Android", + "model": "AOSP on IA Emulator", + "manufacturer": "Google", + "adTrackingEnabled": true, + "advertisingId": "44c97318-9040-4361-8bc7-4eb30f665ca8" + }, + "traits": { + "email": "alex@example.com", + "phone": "+1-202-555-0146", + "firstName": "John", + "lastName": "Gomes", + "city": "London", + "state": "England", + "countryCode": "GB", + "postalCode": "EC3M", + "streetAddress": "71 Cherry Court SOUTHAMPTON SO53 5PD UK" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, + "properties": { + "profileId": "34245", + "floodlightConfigurationId": "213123123", + "ordinal": "string", + "floodlightActivityId": "456543345245", + "value": "756", + "quantity": "455678", + "timestamp": "1668624722903000", + "matchId": "12312321", + "encryptionSource": "AD_SERVING", + "encryptionEntityId": "3564523", + "encryptionEntityType": "DCM_ACCOUNT", + "requestType": "batchupdate" + }, + "originalTimestamp": 1668624722555000, + "type": "track", + "event": "event test", + "anonymousId": "randomId", + "integrations": { + "All": true + }, + "name": "ApplicationLoaded", + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "secret": { + "access_token": "dummyApiToken", + "refresh_token": "efgh5678", + "developer_token": "ijkl91011" + } + }, + "destination": { + "Config": { + "profileId": "5343234", + "treatmentForUnderage": false, + "limitAdTracking": false, + "childDirectedTreatment": false, + "nonPersonalizedAd": false, + "rudderAccountId": "2EOknn1JNH7WK1MfNku4fGYKkRK" + } + } + }, + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate", + "headers": { + "Authorization": "Bearer dummyApiToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "kind": "dfareporting#conversionsBatchUpdateRequest", + "conversions": [ + { + "floodlightConfigurationId": "213123123", + "ordinal": "string", + "timestampMicros": "1668624722555000", + "floodlightActivityId": "456543345245", + "quantity": "455678", + "value": 756, + "matchId": "12312321", + "treatmentForUnderage": false, + "nonPersonalizedAd": false + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + }, + { + "description": "Track - batch insert Call failure", + "input": { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "device": { + "id": "0572f78fa49c648e", + "name": "generic_x86_arm", + "type": "Android", + "model": "AOSP on IA Emulator", + "manufacturer": "Google", + "adTrackingEnabled": true, + "advertisingId": "44c97318-9040-4361-8bc7-4eb30f665ca8" + }, + "traits": { + "email": "alex@example.com", + "phone": "+1-202-555-0146", + "firstName": "John", + "lastName": "Gomes", + "city": "London", + "state": "England", + "countryCode": "GB", + "postalCode": "EC3M", + "streetAddress": "71 Cherry Court SOUTHAMPTON SO53 5PD UK" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, + "originalTimestamp": "2022-11-17T00:22:02.903+05:30", + "properties": { + "profileId": "34245", + "floodlightConfigurationId": "213123123", + "ordinal": "string", + "floodlightActivityId": "456543345245", + "value": "756", + "quantity": "455678", + "requestType": "batchinsert" + }, + "type": "track", + "event": "event test", + "anonymousId": "randomId", + "integrations": { + "All": true + }, + "name": "ApplicationLoaded", + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "secret": { + "access_token": "dummyApiToken", + "refresh_token": "efgh5678", + "developer_token": "ijkl91011" + } + }, + "destination": { + "Config": { + "profileId": "5343234", + "treatmentForUnderage": false, + "limitAdTracking": false, + "childDirectedTreatment": false, + "nonPersonalizedAd": false, + "rudderAccountId": "2EOknn1JNH7WK1MfNku4fGYKkRK" + } + } + }, + "output": { + "statusCode": 400, + "error": "[CAMPAIGN MANAGER (DCM)]: Atleast one of encryptedUserId,encryptedUserIdCandidates, matchId, mobileDeviceId, gclid, dclid, impressionId.", + "statTags": { + "destination": "blueshift", + "stage": "transform", + "scope": "exception" + } + } + }, + { + "description": "Track - batch update Call timestamp as unix", + "input": { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "device": { + "id": "0572f78fa49c648e", + "name": "generic_x86_arm", + "type": "Android", + "model": "AOSP on IA Emulator", + "manufacturer": "Google", + "adTrackingEnabled": true, + "advertisingId": "44c97318-9040-4361-8bc7-4eb30f665ca8" + }, + "traits": { + "email": "alex@example.com", + "phone": "+1-202-555-0146", + "firstName": "John", + "lastName": "Gomes", + "city": "London", + "state": "England", + "countryCode": "GB", + "postalCode": "EC3M", + "streetAddress": "71 Cherry Court SOUTHAMPTON SO53 5PD UK" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, + "properties": { + "profileId": "34245", + "floodlightConfigurationId": "213123123", + "ordinal": "string", + "floodlightActivityId": "456543345245", + "value": "756", + "quantity": "455678", + "timestamp": "1668624722903000", + "matchId": "12312321", + "requestType": "batchupdate" + }, + "originalTimestamp": 1668624722555000, + "type": "track", + "event": "event test", + "anonymousId": "randomId", + "integrations": { + "All": true + }, + "name": "ApplicationLoaded", + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "secret": { + "access_token": "dummyApiToken", + "refresh_token": "efgh5678", + "developer_token": "ijkl91011" + } + }, + "destination": { + "Config": { + "profileId": "5343234", + "treatmentForUnderage": false, + "limitAdTracking": false, + "childDirectedTreatment": false, + "nonPersonalizedAd": false, + "rudderAccountId": "2EOknn1JNH7WK1MfNku4fGYKkRK" + } + } + }, + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate", + "headers": { + "Authorization": "Bearer dummyApiToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "kind": "dfareporting#conversionsBatchUpdateRequest", + "conversions": [ + { + "floodlightConfigurationId": "213123123", + "ordinal": "string", + "timestampMicros": "1668624722555000", + "floodlightActivityId": "456543345245", + "quantity": "455678", + "value": 756, + "matchId": "12312321", + "treatmentForUnderage": false, + "nonPersonalizedAd": false + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } } ] diff --git a/test/__tests__/data/campaign_manager_router_output.json b/test/__tests__/data/campaign_manager_router_output.json index e3a9a39c28..61c3438ab6 100644 --- a/test/__tests__/data/campaign_manager_router_output.json +++ b/test/__tests__/data/campaign_manager_router_output.json @@ -23,7 +23,7 @@ { "nonPersonalizedAd": false, "treatmentForUnderage": false, - "timestampMicros": "1668624722000000", + "timestampMicros": "1668624722903000", "floodlightConfigurationId": "213123123", "ordinal": "string", "quantity": "455678", @@ -87,7 +87,7 @@ { "nonPersonalizedAd": false, "treatmentForUnderage": false, - "timestampMicros": "1668624722000000", + "timestampMicros": "1668624722903000", "floodlightConfigurationId": "213123123", "ordinal": "string", "quantity": "455678", From 3771b3dc24f9a5d8c5e5e1a781c823e923b810dd Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 25 Sep 2023 17:58:17 +0530 Subject: [PATCH 2/5] feat: batching in cm360 --- .../destinations/campaign_manager/config.js | 3 + .../campaign_manager/transform.js | 122 +++++++++++++++++- test/__tests__/campaign_manager.test.js | 5 +- .../data/campaign_manager_router_input.json | 94 +++++++++++++- .../data/campaign_manager_router_output.json | 27 +++- 5 files changed, 232 insertions(+), 19 deletions(-) diff --git a/src/v0/destinations/campaign_manager/config.js b/src/v0/destinations/campaign_manager/config.js index 063f65cb14..b3a9531347 100644 --- a/src/v0/destinations/campaign_manager/config.js +++ b/src/v0/destinations/campaign_manager/config.js @@ -9,6 +9,8 @@ const ConfigCategories = { }, }; +const MAX_BATCH_CONVERSATIONS_SIZE = 1000; + const EncryptionEntityType = [ 'ENCRYPTION_ENTITY_TYPE_UNKNOWN', 'DCM_ACCOUNT', @@ -28,4 +30,5 @@ module.exports = { BASE_URL, EncryptionEntityType, EncryptionSource, + MAX_BATCH_CONVERSATIONS_SIZE, }; diff --git a/src/v0/destinations/campaign_manager/transform.js b/src/v0/destinations/campaign_manager/transform.js index 723980f0e4..a8983ebccd 100644 --- a/src/v0/destinations/campaign_manager/transform.js +++ b/src/v0/destinations/campaign_manager/transform.js @@ -1,11 +1,15 @@ +const lodash = require('lodash'); const { EventType } = require('../../../constants'); const { constructPayload, defaultRequestConfig, defaultPostRequestConfig, + defaultBatchRequestConfig, removeUndefinedAndNullValues, + getSuccessRespEvents, isDefinedAndNotNull, - simpleProcessRouterDest, + checkInvalidRtTfEvents, + handleRtTfSingleEventError, getAccessToken, } = require('../../util'); @@ -15,6 +19,7 @@ const { BASE_URL, EncryptionEntityType, EncryptionSource, + MAX_BATCH_CONVERSATIONS_SIZE, } = require('./config'); const { InstrumentationError } = require('../../util/errorTypes'); @@ -75,10 +80,12 @@ function processTrack(message, metadata, destination) { const date = new Date(requestJson.timestampMicros); let unixTimestamp = date.getTime(); // Date, moment both are not able to distinguish input if it is second,millisecond or microsecond unix timestamp - // Using count of digits to distinguish between the 3, 9999999999999 (13 digits) means Nov 20 2286 - if (unixTimestamp.toString().length === 13) { // milliseconds + // Using count of digits to distinguish between the 3, 9999999999999 (13 digits) means Nov 20 2286 + if (unixTimestamp.toString().length === 13) { + // milliseconds unixTimestamp *= 1000; - } else if (unixTimestamp.toString().length === 10) { // seconds + } else if (unixTimestamp.toString().length === 10) { + // seconds unixTimestamp *= 1000000; } requestJson.timestampMicros = unixTimestamp.toString(); @@ -147,7 +154,7 @@ function postValidateRequest(response) { ); } - if ( + if ( !response.body.JSON.conversions[0].gclid && !response.body.JSON.conversions[0].matchId && !response.body.JSON.conversions[0].dclid && @@ -185,9 +192,110 @@ function process(event) { return response; } +const generateBatch = (eventKind, events) => { + const batchRequestObject = defaultBatchRequestConfig(); + const conversions = []; + let encryptionInfo = {}; + const metadata = []; + // extracting destination, message from the first event in a batch + const { destination, message } = events[0]; + // Batch event into dest batch structure + events.forEach((ev) => { + conversions.push(...ev.message.body.JSON.conversions); + metadata.push(ev.metadata); + if (ev.message.body.JSON.encryptionInfo) { + encryptionInfo = ev.message.body.JSON.encryptionInfo; + } + }); + + batchRequestObject.batchedRequest.body.JSON = { + kind: eventKind, + conversions, + }; + + if (Object.keys(encryptionInfo).length > 0) { + batchRequestObject.batchedRequest.body.JSON.encryptionInfo = encryptionInfo; + } + + batchRequestObject.batchedRequest.endpoint = message.endpoint; + + batchRequestObject.batchedRequest.headers = message.headers; + + return { + ...batchRequestObject, + metadata, + destination, + }; +}; + +const batchEvents = (eventChunksArray) => { + const batchedResponseList = []; + + // group batchInsert and batchUpdate payloads + const groupedEventChunks = lodash.groupBy( + eventChunksArray, + (event) => event.message.body.JSON.kind, + ); + Object.keys(groupedEventChunks).forEach((eventKind) => { + // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] + const eventChunks = lodash.chunk(groupedEventChunks[eventKind], MAX_BATCH_CONVERSATIONS_SIZE); + eventChunks.forEach((chunk) => { + const batchEventResponse = generateBatch(eventKind, chunk); + batchedResponseList.push( + getSuccessRespEvents( + batchEventResponse.batchedRequest, + batchEventResponse.metadata, + batchEventResponse.destination, + true, + ), + ); + }); + }); + return batchedResponseList; +}; + const processRouterDest = async (inputs, reqMetadata) => { - const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); - return respList; + const errorRespEvents = checkInvalidRtTfEvents(inputs); + if (errorRespEvents.length > 0) { + return errorRespEvents; + } + + const batchErrorRespList = []; + const eventChunksArray = []; + const { destination } = inputs[0]; + await Promise.all( + inputs.map(async (event) => { + try { + if (event.message.statusCode) { + // already transformed event + eventChunksArray.push({ + message: event.message, + metadata: event.metadata, + destination, + }); + } else { + // if not transformed + const proccessedRespList = process(event); + const transformedPayload = { + message: proccessedRespList, + metadata: event.metadata, + destination, + }; + eventChunksArray.push(transformedPayload); + } + } catch (error) { + const errRespEvent = handleRtTfSingleEventError(event, error, reqMetadata); + batchErrorRespList.push(errRespEvent); + } + }), + ); + + let batchResponseList = []; + if (eventChunksArray.length > 0) { + batchResponseList = batchEvents(eventChunksArray); + } + + return [...batchResponseList, ...batchErrorRespList]; }; module.exports = { process, processRouterDest }; diff --git a/test/__tests__/campaign_manager.test.js b/test/__tests__/campaign_manager.test.js index 4f3a40d520..15568eddb2 100644 --- a/test/__tests__/campaign_manager.test.js +++ b/test/__tests__/campaign_manager.test.js @@ -28,8 +28,7 @@ describe(`${name} Tests`, () => { testData.forEach((dataPoint, index) => { it(`${index}. ${integration} - ${dataPoint.description}`, async () => { try { - let output = await transformer.process(dataPoint.input); - delete output.body.JSON.idempotency; + const output = await transformer.process(dataPoint.input); expect(output).toEqual(dataPoint.output); } catch (error) { expect(error.message).toEqual(dataPoint.output.error); @@ -37,7 +36,7 @@ describe(`${name} Tests`, () => { }); }); }); - + describe("Router Tests", () => { it("Payload", async () => { const routerOutput = await transformer.processRouterDest(inputRouterData); diff --git a/test/__tests__/data/campaign_manager_router_input.json b/test/__tests__/data/campaign_manager_router_input.json index 18893a0b43..8cae574c55 100644 --- a/test/__tests__/data/campaign_manager_router_input.json +++ b/test/__tests__/data/campaign_manager_router_input.json @@ -61,8 +61,6 @@ "density": 2 } }, - "event": "Promotion Clicked", - "type": "track", "originalTimestamp": "2022-11-17T00:22:02.903+05:30", "properties": { "profileId": 437689, @@ -70,7 +68,7 @@ "ordinal": "string", "quantity": "455678", "floodlightActivityId": "456543345245", - "value": 7, + "value": 47, "encryptedUserIdCandidates": ["dfghjbnm"], "limitAdTracking": true, "childDirectedTreatment": true, @@ -151,8 +149,94 @@ "density": 2 } }, - "event": "Promotion Clicked", + "originalTimestamp": "2023-11-17T00:22:02.903+05:30", + "properties": { + "profileId": 437689, + "floodlightConfigurationId": "213123123", + "ordinal": "345345345", + "quantity": "4556784325345345", + "floodlightActivityId": "456543345245", + "value": 17, + "encryptedUserIdCandidates": ["dfghjbnm"], + "limitAdTracking": true, + "childDirectedTreatment": true, + "encryptionSource": "AD_SERVING", + "encryptionEntityId": "3564523", + "encryptionEntityType": "DCM_ACCOUNT", + "requestType": "batchinsert" + }, "type": "track", + "event": "event test", + "anonymousId": "randomId", + "integrations": { + "All": true + }, + "name": "ApplicationLoaded", + "sentAt": "2023-11-17T00:22:02.903+05:30" + } + }, + { + "metadata": { + "secret": { + "access_token": "dummyApiToken", + "refresh_token": "efgh5678", + "developer_token": "ijkl91011" + }, + "jobId": 2 + }, + "destination": { + "Config": { + "treatmentForUnderage": false, + "limitAdTracking": false, + "childDirectedTreatment": false, + "nonPersonalizedAd": false, + "rudderAccountId": "2EOknn1JNH7WK1MfNku4fGYKkRK" + } + }, + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "device": { + "id": "0572f78fa49c648e", + "name": "generic_x86_arm", + "type": "Android", + "model": "AOSP on IA Emulator", + "manufacturer": "Google", + "adTrackingEnabled": true, + "advertisingId": "44c97318-9040-4361-8bc7-4eb30f665ca8" + }, + "traits": { + "email": "alex@example.com", + "phone": "+1-202-555-0146", + "firstName": "John", + "lastName": "Gomes", + "city": "London", + "state": "England", + "countryCode": "GB", + "postalCode": "EC3M", + "streetAddress": "71 Cherry Court SOUTHAMPTON SO53 5PD UK" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, "originalTimestamp": "2022-11-17T00:22:02.903+05:30", "properties": { "profileId": 437689, @@ -241,8 +325,6 @@ "density": 2 } }, - "event": "Promotion Clicked", - "type": "track", "originalTimestamp": "2022-11-17T00:22:02.903+05:30", "properties": { "profileId": 437689, diff --git a/test/__tests__/data/campaign_manager_router_output.json b/test/__tests__/data/campaign_manager_router_output.json index 61c3438ab6..f92713e2b3 100644 --- a/test/__tests__/data/campaign_manager_router_output.json +++ b/test/__tests__/data/campaign_manager_router_output.json @@ -28,10 +28,23 @@ "ordinal": "string", "quantity": "455678", "floodlightActivityId": "456543345245", - "value": 7, + "value": 47, "encryptedUserIdCandidates": ["dfghjbnm"], "limitAdTracking": true, "childDirectedTreatment": true + }, + { + "floodlightConfigurationId": "213123123", + "ordinal": "345345345", + "quantity": "4556784325345345", + "floodlightActivityId": "456543345245", + "value": 17, + "encryptedUserIdCandidates": ["dfghjbnm"], + "limitAdTracking": true, + "childDirectedTreatment": true, + "nonPersonalizedAd": false, + "timestampMicros": "1700160722903000", + "treatmentForUnderage": false } ] }, @@ -49,9 +62,17 @@ "refresh_token": "efgh5678" }, "jobId": 1 + }, + { + "jobId": 2, + "secret": { + "access_token": "dummyApiToken", + "developer_token": "ijkl91011", + "refresh_token": "efgh5678" + } } ], - "batched": false, + "batched": true, "statusCode": 200, "destination": { "Config": { @@ -113,7 +134,7 @@ "jobId": 2 } ], - "batched": false, + "batched": true, "statusCode": 200, "destination": { "Config": { From 3e589e70e72e70436899b39e379d0419b1a034b6 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Tue, 26 Sep 2023 16:30:04 +0530 Subject: [PATCH 3/5] chore: addressed comments --- .../data/CampaignManagerTrackConfig.json | 3 +- .../campaign_manager/networkHandler.js | 34 ++++- .../campaign_manager/transform.js | 11 +- test/__tests__/data/campaign_manager.json | 126 +++++++++++------- 4 files changed, 121 insertions(+), 53 deletions(-) diff --git a/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json b/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json index f36ba8da8a..c8bd10d524 100644 --- a/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json +++ b/src/v0/destinations/campaign_manager/data/CampaignManagerTrackConfig.json @@ -12,7 +12,8 @@ { "destKey": "timestampMicros", "sourceKeys": "timestamp", - "sourceFromGenericMap": true + "sourceFromGenericMap": true, + "required": true }, { "destKey": "floodlightActivityId", diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index a80fff6fe4..d8498b7621 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -9,14 +9,36 @@ const { AbortedError, RetryableError, NetworkError } = require('../../util/error const tags = require('../../util/tags'); function checkIfFailuresAreRetryable(response) { + const { status } = response; + let isRetryable = true; try { - if (Array.isArray(response.status) && Array.isArray(response.status[0].errors)) { - return ( - response.status[0].errors[0].code !== 'PERMISSION_DENIED' && - response.status[0].errors[0].code !== 'INVALID_ARGUMENT' - ); + if (Array.isArray(status)) { + // iterate over each status, and if found retryable in conversations ..retry else discard + /* status : [{ + "conversion": { + object (Conversion) + }, + "errors": [ + { + object (ConversionError) + } + ], + "kind": string + }] */ + for (const st of status) { + st.errors.forEach(err => { + // if code is any of these, event is not retryable + if ( + err.code === 'PERMISSION_DENIED' || + err.code === 'INVALID_ARGUMENT' || + err.code === 'NOT_FOUND' + ) { + isRetryable = false; + } + }) + } } - return true; + return isRetryable; } catch (e) { return true; } diff --git a/src/v0/destinations/campaign_manager/transform.js b/src/v0/destinations/campaign_manager/transform.js index a8983ebccd..5393f18631 100644 --- a/src/v0/destinations/campaign_manager/transform.js +++ b/src/v0/destinations/campaign_manager/transform.js @@ -77,10 +77,19 @@ function processTrack(message, metadata, destination) { delete requestJson.limitAdTracking; } + // for handling when input is timestamp as string + const numTimestamp = /^\d+$/.test(requestJson.timestampMicros); + if (numTimestamp) { + // is digit only, below convert string timestamp to numeric + requestJson.timestampMicros *= 1; + } + + // 2022-10-11T05:453:90.ZZ + // 16483423423423423 const date = new Date(requestJson.timestampMicros); let unixTimestamp = date.getTime(); // Date, moment both are not able to distinguish input if it is second,millisecond or microsecond unix timestamp - // Using count of digits to distinguish between the 3, 9999999999999 (13 digits) means Nov 20 2286 + // Using count of digits to distinguish between these 3, 9999999999999 (13 digits) means Nov 20 2286 which is long far in future if (unixTimestamp.toString().length === 13) { // milliseconds unixTimestamp *= 1000; diff --git a/test/__tests__/data/campaign_manager.json b/test/__tests__/data/campaign_manager.json index 6139ddbeb3..4347ce8394 100644 --- a/test/__tests__/data/campaign_manager.json +++ b/test/__tests__/data/campaign_manager.json @@ -572,53 +572,89 @@ "description": "Track - batch update Call timestamp as unix", "input": { "message": { - "channel": "web", - "context": { - "app": { - "build": "1.0.0", - "name": "RudderLabs JavaScript SDK", - "namespace": "com.rudderlabs.javascript", - "version": "1.0.0" - }, - "device": { - "id": "0572f78fa49c648e", - "name": "generic_x86_arm", - "type": "Android", - "model": "AOSP on IA Emulator", - "manufacturer": "Google", - "adTrackingEnabled": true, - "advertisingId": "44c97318-9040-4361-8bc7-4eb30f665ca8" - }, - "traits": { - "email": "alex@example.com", - "phone": "+1-202-555-0146", - "firstName": "John", - "lastName": "Gomes", - "city": "London", - "state": "England", - "countryCode": "GB", - "postalCode": "EC3M", - "streetAddress": "71 Cherry Court SOUTHAMPTON SO53 5PD UK" - }, - "library": { - "name": "RudderLabs JavaScript SDK", - "version": "1.0.0" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - "locale": "en-US", - "ip": "0.0.0.0", - "os": { - "name": "", - "version": "" - }, - "screen": { - "density": 2 - } + "channel": "sources", + "properties": { + "profileId": "34245", + "floodlightConfigurationId": "213123123", + "ordinal": "213123", + "floodlightActivityId": "456543345245", + "value": "756", + "quantity": "455678", + "timestamp": "1668624722903000", + "matchId": "12312321", + "requestType": "batchupdate" + }, + "timestamp": "1668624722555000", + "type": "track", + "event": "event test", + "anonymousId": "randomId", + "integrations": { + "All": true }, + "name": "ApplicationLoaded", + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "secret": { + "access_token": "dummyApiToken", + "refresh_token": "efgh5678", + "developer_token": "ijkl91011" + } + }, + "destination": { + "Config": { + "profileId": "5343234", + "treatmentForUnderage": false, + "limitAdTracking": false, + "childDirectedTreatment": false, + "nonPersonalizedAd": false, + "rudderAccountId": "2EOknn1JNH7WK1MfNku4fGYKkRK" + } + } + }, + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate", + "headers": { + "Authorization": "Bearer dummyApiToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "kind": "dfareporting#conversionsBatchUpdateRequest", + "conversions": [ + { + "floodlightConfigurationId": "213123123", + "ordinal": "213123", + "timestampMicros": "1668624722555000", + "floodlightActivityId": "456543345245", + "quantity": "455678", + "value": 756, + "matchId": "12312321", + "treatmentForUnderage": false, + "nonPersonalizedAd": false + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + }, + { + "description": "Track - batch update Call timestamp as unix as digit", + "input": { + "message": { + "channel": "sources", "properties": { "profileId": "34245", "floodlightConfigurationId": "213123123", - "ordinal": "string", + "ordinal": "213123", "floodlightActivityId": "456543345245", "value": "756", "quantity": "455678", @@ -626,7 +662,7 @@ "matchId": "12312321", "requestType": "batchupdate" }, - "originalTimestamp": 1668624722555000, + "timestamp": 1668624722555000, "type": "track", "event": "event test", "anonymousId": "randomId", @@ -670,7 +706,7 @@ "conversions": [ { "floodlightConfigurationId": "213123123", - "ordinal": "string", + "ordinal": "213123", "timestampMicros": "1668624722555000", "floodlightActivityId": "456543345245", "quantity": "455678", From 44f10483208c8e9e65d544de791ec33808f51582 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Tue, 26 Sep 2023 18:18:48 +0530 Subject: [PATCH 4/5] chore: addressed comments --- src/v0/destinations/campaign_manager/networkHandler.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index d8498b7621..d094567fb2 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -10,7 +10,6 @@ const tags = require('../../util/tags'); function checkIfFailuresAreRetryable(response) { const { status } = response; - let isRetryable = true; try { if (Array.isArray(status)) { // iterate over each status, and if found retryable in conversations ..retry else discard @@ -26,19 +25,19 @@ function checkIfFailuresAreRetryable(response) { "kind": string }] */ for (const st of status) { - st.errors.forEach(err => { + for(const err of st.errors) { // if code is any of these, event is not retryable if ( err.code === 'PERMISSION_DENIED' || err.code === 'INVALID_ARGUMENT' || err.code === 'NOT_FOUND' ) { - isRetryable = false; + return false; } - }) + } } } - return isRetryable; + return true; } catch (e) { return true; } From 0cf895e309ad032770b5ab010a124bc625f5f089 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Wed, 27 Sep 2023 02:36:27 +0530 Subject: [PATCH 5/5] fix: return aborted inside cath block --- src/v0/destinations/campaign_manager/networkHandler.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index d094567fb2..a8c57902e5 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -39,7 +39,7 @@ function checkIfFailuresAreRetryable(response) { } return true; } catch (e) { - return true; + return false; } } @@ -73,7 +73,7 @@ const responseHandler = (destinationResponse) => { } throw new NetworkError( - `Campaign Manager: ${response.error.message} during CAMPAIGN_MANAGER response transformation 3`, + `Campaign Manager: ${response.error?.message} during CAMPAIGN_MANAGER response transformation`, status, { [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status),