From 850397e408269c7b967a407e05f5b435a80e4b1c Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Fri, 10 Nov 2023 16:17:17 +0530 Subject: [PATCH 01/16] feat: initial commit --- src/v0/destinations/mp/transform.js | 57 ++++++++++++++++++++++++++--- src/v0/destinations/mp/util.js | 5 +++ 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index a5f1bf2d9d..83041125a0 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -34,6 +34,7 @@ const { combineBatchRequestsWithSameJobIds, groupEventsByEndpoint, batchEvents, + parseConfigArray } = require('./util'); const { InstrumentationError, ConfigurationError } = require('../../util/errorTypes'); const { CommonUtils } = require('../../../util/common'); @@ -106,6 +107,13 @@ const responseBuilderSimple = (payload, message, eventType, destConfig) => { strict: credentials.params.strict, }; break; + // case 'setOnce': + // response.endpoint = dataResidency === 'eu' ? `${BASE_ENDPOINT_EU}/engage#profile-set-once` : `${BASE_ENDPOINT}/engage#profile-set-once`; + // response.headers = { + // 'Content-Type': 'application/json', + // 'accept': 'text/plain', + // }; + // break; default: response.endpoint = dataResidency === 'eu' ? `${BASE_ENDPOINT_EU}/engage/` : `${BASE_ENDPOINT}/engage/`; @@ -226,17 +234,54 @@ const processTrack = (message, destination) => { return returnValue; }; +function trimTraits(traits, setOnceProperties) { + // Create a copy of the original traits object + const traitsCopy = { ...traits }; + + // Initialize setOnce object + const setOnce = {}; + + // Iterate over setOnceProperties and move corresponding properties to setOnce + setOnceProperties.forEach(property => { + if (traitsCopy.hasOwnProperty(property)) { + setOnce[property] = traitsCopy[property]; + delete traitsCopy[property]; + } + }); + + return { + traits: traitsCopy, + setOnce, + }; +} + +const createSetOnceResponse = (message, type, destination, setOnce) => { + const payload = { + $set_once: setOnce, + $token: destination.Config.token, + $distinct_id: message.userId || message.anonymousId, + }; + return responseBuilderSimple(payload, message, type, destination.Config); +} + + + const processIdentifyEvents = async (message, type, destination) => { + const messageClone = { ...message }; + const setOnceProperties = parseConfigArray(destination.Config.setOnceProperties, 'property'); + const { traits, setOnce } = trimTraits(getFieldValueFromMessage(messageClone, 'traits'), setOnceProperties); + messageClone.traits = traits; + const returnValue = []; - // Creating the user profile // https://developer.mixpanel.com/reference/profile-set - returnValue.push(createIdentifyResponse(message, type, destination, responseBuilderSimple)); + returnValue.push(createIdentifyResponse(messageClone, type, destination, responseBuilderSimple)); + returnValue.push(createSetOnceResponse(messageClone, type, destination, setOnce)); if ( destination.Config?.identityMergeApi !== 'simplified' && - message.userId && - message.anonymousId && + messageClone.userId && + messageClone.anonymousId && isImportAuthCredentialsAvailable(destination) ) { // If userId and anonymousId both are present and required credentials for /import @@ -245,13 +290,13 @@ const processIdentifyEvents = async (message, type, destination) => { const trackPayload = { event: '$merge', properties: { - $distinct_ids: [message.userId, message.anonymousId], + $distinct_ids: [messageClone.userId, messageClone.anonymousId], token: destination.Config.token, }, }; const identifyTrackResponse = responseBuilderSimple( trackPayload, - message, + messageClone, 'merge', destination.Config, ); diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index 30173fd514..bdfbaf0a39 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -322,6 +322,10 @@ const combineBatchRequestsWithSameJobIds = (inputBatches) => { return combineBatches(combineBatches(inputBatches)); }; +const parseConfigArray = (arr, key) => + arr.map(item => item[key]) +; + module.exports = { createIdentifyResponse, isImportAuthCredentialsAvailable, @@ -330,4 +334,5 @@ module.exports = { generateBatchedPayloadForArray, batchEvents, combineBatchRequestsWithSameJobIds, + parseConfigArray }; From 55715dd08a2899e47ad9979bde6eb8808c1ccfcc Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Fri, 10 Nov 2023 17:15:28 +0530 Subject: [PATCH 02/16] feat: making sure existing functionality is intact --- src/v0/destinations/mp/transform.js | 31 +++++++++++++++++++---------- src/v0/destinations/mp/util.js | 11 ++++++---- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index d188754ea1..137f8f8e97 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -35,7 +35,7 @@ const { combineBatchRequestsWithSameJobIds, groupEventsByEndpoint, batchEvents, - parseConfigArray + parseConfigArray, } = require('./util'); const { CommonUtils } = require('../../../util/common'); @@ -242,7 +242,7 @@ function trimTraits(traits, setOnceProperties) { const setOnce = {}; // Iterate over setOnceProperties and move corresponding properties to setOnce - setOnceProperties.forEach(property => { + setOnceProperties.forEach((property) => { if (traitsCopy.hasOwnProperty(property)) { setOnce[property] = traitsCopy[property]; delete traitsCopy[property]; @@ -262,21 +262,32 @@ const createSetOnceResponse = (message, type, destination, setOnce) => { $distinct_id: message.userId || message.anonymousId, }; return responseBuilderSimple(payload, message, type, destination.Config); -} - - +}; const processIdentifyEvents = async (message, type, destination) => { const messageClone = { ...message }; - const setOnceProperties = parseConfigArray(destination.Config.setOnceProperties, 'property'); - const { traits, setOnce } = trimTraits(getFieldValueFromMessage(messageClone, 'traits'), setOnceProperties); - messageClone.traits = traits; - + let seggregatedTraits = {}; const returnValue = []; + + // making payload for set_once properties + if ( + destination.Config.setOnceProperties && + Object.keys(destination.Config.setOnceProperties).length > 0 + ) { + const setOnceProperties = parseConfigArray(destination.Config.setOnceProperties, 'property'); + seggregatedTraits = trimTraits( + getFieldValueFromMessage(messageClone, 'traits'), + setOnceProperties, + ); + messageClone.traits = seggregatedTraits.traits; + returnValue.push( + createSetOnceResponse(messageClone, type, destination, seggregatedTraits.setOnce), + ); + } + // Creating the user profile // https://developer.mixpanel.com/reference/profile-set returnValue.push(createIdentifyResponse(messageClone, type, destination, responseBuilderSimple)); - returnValue.push(createSetOnceResponse(messageClone, type, destination, setOnce)); if ( destination.Config?.identityMergeApi !== 'simplified' && diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index 4863f51f2b..4acc7824fc 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -322,9 +322,12 @@ const combineBatchRequestsWithSameJobIds = (inputBatches) => { return combineBatches(combineBatches(inputBatches)); }; -const parseConfigArray = (arr, key) => - arr.map(item => item[key]) -; +const parseConfigArray = (arr, key) => { + if (!arr) { + return []; + } + return arr.map((item) => item[key]); +}; module.exports = { createIdentifyResponse, @@ -334,5 +337,5 @@ module.exports = { generateBatchedPayloadForArray, batchEvents, combineBatchRequestsWithSameJobIds, - parseConfigArray + parseConfigArray, }; From 682fc95f35b661677553ac925877467160d543ae Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 13 Nov 2023 11:57:34 +0530 Subject: [PATCH 03/16] fix: edits for exclusion keys --- .../mp/data/MPIdentifyConfig.json | 12 +- src/v0/destinations/mp/transform.js | 26 +- src/v0/destinations/mp/util.js | 13 +- test/integrations/destinations/mp/common.ts | 40 ++- .../destinations/mp/processor/data.ts | 262 +++++++++++++++++- 5 files changed, 340 insertions(+), 13 deletions(-) diff --git a/src/v0/destinations/mp/data/MPIdentifyConfig.json b/src/v0/destinations/mp/data/MPIdentifyConfig.json index 679ce66d7f..44c99fb56d 100644 --- a/src/v0/destinations/mp/data/MPIdentifyConfig.json +++ b/src/v0/destinations/mp/data/MPIdentifyConfig.json @@ -13,7 +13,17 @@ }, { "destKey": "$first_name", - "sourceKeys": "firstName", + "sourceKeys": [ + "firstName", + "firstname", + "first_name", + "traits.firstName", + "traits.firstname", + "traits.first_name", + "context.traits.firstName", + "context.traits.firstname", + "context.traits.first_name" + ], "required": false, "sourceFromGenericMap": true }, diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 137f8f8e97..9cf43b88f7 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -41,6 +41,7 @@ const { CommonUtils } = require('../../../util/common'); // ref: https://help.mixpanel.com/hc/en-us/articles/115004613766-Default-Properties-Collected-by-Mixpanel const mPEventPropertiesConfigJson = mappingConfig[ConfigCategory.EVENT_PROPERTIES.name]; +const mPIdentifyConfigJson = mappingConfig[ConfigCategory.IDENTIFY.name]; const setImportCredentials = (destConfig) => { const endpoint = @@ -234,9 +235,10 @@ const processTrack = (message, destination) => { return returnValue; }; -function trimTraits(traits, setOnceProperties) { +function trimTraits(traits, contextTraits, setOnceProperties) { // Create a copy of the original traits object const traitsCopy = { ...traits }; + const contextTraitsCopy = { ...contextTraits }; // Initialize setOnce object const setOnce = {}; @@ -247,11 +249,20 @@ function trimTraits(traits, setOnceProperties) { setOnce[property] = traitsCopy[property]; delete traitsCopy[property]; } + if (contextTraitsCopy.hasOwnProperty(property)) { + if(!setOnce.hasOwnProperty(property)) { + setOnce[property] = contextTraitsCopy[property]; + } + delete contextTraitsCopy[property]; + } }); + const sentOnceTransform = constructPayload(setOnce, mPIdentifyConfigJson); + return { traits: traitsCopy, - setOnce, + contextTraits: contextTraitsCopy, + sentOnceTransform, }; } @@ -268,18 +279,21 @@ const processIdentifyEvents = async (message, type, destination) => { const messageClone = { ...message }; let seggregatedTraits = {}; const returnValue = []; + let setOnceProperties = []; // making payload for set_once properties if ( destination.Config.setOnceProperties && Object.keys(destination.Config.setOnceProperties).length > 0 ) { - const setOnceProperties = parseConfigArray(destination.Config.setOnceProperties, 'property'); + setOnceProperties = parseConfigArray(destination.Config.setOnceProperties, 'property'); seggregatedTraits = trimTraits( - getFieldValueFromMessage(messageClone, 'traits'), + messageClone.traits, + messageClone.context.traits, setOnceProperties, ); messageClone.traits = seggregatedTraits.traits; + messageClone.context.traits = seggregatedTraits.contextTraits; returnValue.push( createSetOnceResponse(messageClone, type, destination, seggregatedTraits.setOnce), ); @@ -287,7 +301,7 @@ const processIdentifyEvents = async (message, type, destination) => { // Creating the user profile // https://developer.mixpanel.com/reference/profile-set - returnValue.push(createIdentifyResponse(messageClone, type, destination, responseBuilderSimple)); + returnValue.push(createIdentifyResponse(messageClone, type, destination, responseBuilderSimple, setOnceProperties)); if ( destination.Config?.identityMergeApi !== 'simplified' && @@ -496,7 +510,7 @@ const processRouterDest = async (inputs, reqMetadata) => { destination: event.destination, }; } - + // console.log('event', JSON.stringify(event)); let processedEvents = await process(event); processedEvents = CommonUtils.toArray(processedEvents); return processedEvents.map((res) => ({ diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index 4acc7824fc..dd4bbccc93 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -34,7 +34,7 @@ const mPProfileIosConfigJson = mappingConfig[ConfigCategory.PROFILE_IOS.name]; * @param {*} useNewMapping a variable to support backward compatibility * @returns */ -const getTransformedJSON = (message, mappingJson, useNewMapping) => { +const getTransformedJSON = (message, mappingJson, useNewMapping, setOnceProperties) => { let rawPayload = constructPayload(message, mappingJson); if ( isDefined(rawPayload.$geo_source) && @@ -47,11 +47,16 @@ const getTransformedJSON = (message, mappingJson, useNewMapping) => { set(rawPayload, '$name', getFullName(message)); } + let newExclusionList = [...MP_IDENTIFY_EXCLUSION_LIST]; + if (setOnceProperties.length > 0) { + newExclusionList = [...newExclusionList, ...setOnceProperties]; + } + rawPayload = extractCustomFields( message, rawPayload, ['traits', 'context.traits'], - MP_IDENTIFY_EXCLUSION_LIST, + newExclusionList, ); /* @@ -99,11 +104,11 @@ const getTransformedJSON = (message, mappingJson, useNewMapping) => { * @param {*} responseBuilderSimple function to generate response * @returns */ -const createIdentifyResponse = (message, type, destination, responseBuilderSimple) => { +const createIdentifyResponse = (message, type, destination, responseBuilderSimple, setOnceProperties) => { // this variable is used for supporting backward compatibility const { useNewMapping, token } = destination.Config; // user payload created - const properties = getTransformedJSON(message, mPIdentifyConfigJson, useNewMapping); + const properties = getTransformedJSON(message, mPIdentifyConfigJson, useNewMapping, setOnceProperties); const payload = { $set: properties, diff --git a/test/integrations/destinations/mp/common.ts b/test/integrations/destinations/mp/common.ts index ad12566cc6..e2daaa8622 100644 --- a/test/integrations/destinations/mp/common.ts +++ b/test/integrations/destinations/mp/common.ts @@ -20,4 +20,42 @@ const sampleDestination = { Transformations: [], }; -export { sampleDestination, defaultMockFns }; +const destinationWithSetOnceProperty = { + ID: '2Xz4szkZeBMJRgO5TfvLclsxkxz', + Name: 'Mixpanel Dev', + DestinationDefinition: { + ID: '2WcAA63MqbQVSIrgPR0mXuxF4ut', + Name: 'MP', + DisplayName: 'Mixpanel', + ResponseRules: {} + }, + Config: { + apiSecret: 'dummySecret', + dataResidency: 'us', + eventDelivery: false, + eventDeliveryTS: 1699848284554, + identityMergeApi: 'simplified', + setOnceProperties: [ + { + property: 'nationality' + } + ], + superProperties: [ + { + property: 'random' + } + ], + token: 'dummyToken', + useNativeSDK: false, + useNewMapping: false, + userDeletionApi: 'engage', + whitelistedEvents: [] + }, + Enabled: true, + WorkspaceID: 123345, + Transformations: [], + IsProcessorEnabled: true, + RevisionID: 12345 +} + +export { sampleDestination, defaultMockFns, destinationWithSetOnceProperty }; diff --git a/test/integrations/destinations/mp/processor/data.ts b/test/integrations/destinations/mp/processor/data.ts index 2745c09ecc..7038a24fac 100644 --- a/test/integrations/destinations/mp/processor/data.ts +++ b/test/integrations/destinations/mp/processor/data.ts @@ -1,5 +1,5 @@ import { overrideDestination } from '../../../testUtils'; -import { sampleDestination, defaultMockFns } from '../common'; +import { sampleDestination, defaultMockFns, destinationWithSetOnceProperty } from '../common'; export const data = [ { @@ -5852,4 +5852,264 @@ export const data = [ }, }, }, + { + name: 'mp', + description: 'Test Set Once Property', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + description: 'Alias: with property beyond exclusion list', + destination: destinationWithSetOnceProperty, + message: { + anonymousId: 'e6ab2c5e-2cda-44a9-a962-e2f67df78bca', + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.5' + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.5' + }, + locale: 'en-GB', + os: { + name: '', + version: '' + }, + screen: { + density: 2 + }, + traits: { + city: 'Disney', + country: 'USA', + email: 'TestSanity@disney.com', + firstName: 'Mickey test', + lastName: 'VarChange', + createdAt: '2020-01-23T08:54:02.362Z', + nationality: 'USA', + random: 'superProp' + }, + page: { + path: '/destinations/mixpanel', + referrer: '', + search: '', + title: '', + url: 'https://docs.rudderstack.com/destinations/mixpanel', + category: 'destination', + initial_referrer: 'https://docs.rudderstack.com', + initial_referring_domain: 'docs.rudderstack.com' + }, + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36' + }, + integrations: { + All: true + }, + page: { + path: '/destinations/mixpanel', + referrer: '', + search: '', + title: '', + url: 'https://docs.rudderstack.com/destinations/mixpanel', + category: 'destination', + initial_referrer: 'https://docs.rudderstack.com', + initial_referring_domain: 'docs.rudderstack.com' + }, + request_ip: '[::1]:53709', + type: 'identify', + userId: 'Santiy' + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + body: { + FORM: {}, + JSON: {}, + JSON_ARRAY: { + batch: "[{\"$set_once\":{\"nationality\":\"USA\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\"}]", + }, + XML: {}, + }, + endpoint: "https://api.mixpanel.com/engage/", + files: {}, + headers: {}, + method: 'POST', + params: {}, + type: 'REST', + userId: 'Santiy', + version: '1', + }, + statusCode: 200, + }, + { + output: { + body: { + FORM: {}, + JSON: {}, + JSON_ARRAY: { + batch: "[{\"$set\":{\"$created\":\"2020-01-23T08:54:02.362Z\",\"$email\":\"TestSanity@disney.com\",\"$country_code\":\"USA\",\"$city\":\"Disney\",\"$initial_referrer\":\"https://docs.rudderstack.com\",\"$initial_referring_domain\":\"docs.rudderstack.com\",\"$name\":\"Mickey test VarChange\",\"random\":\"superProp\",\"$firstName\":\"Mickey test\",\"$lastName\":\"VarChange\",\"$browser\":\"Chrome\",\"$browser_version\":\"79.0.3945.117\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\",\"$ip\":\"0.0.0.0\",\"$time\":null}]", + }, + XML: {}, + }, + endpoint: 'https://api.mixpanel.com/engage/', + files: {}, + headers: {}, + method: 'POST', + params: {}, + type: 'REST', + userId: 'Santiy', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'mp', + description: 'Test Set Once Property', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + description: 'Alias: with property beyond exclusion list', + destination: destinationWithSetOnceProperty, + message: { + anonymousId: 'e6ab2c5e-2cda-44a9-a962-e2f67df78bca', + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.5' + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.5' + }, + locale: 'en-GB', + os: { + name: '', + version: '' + }, + screen: { + density: 2 + }, + traits: { + city: 'Disney', + country: 'USA', + email: 'TestSanity@disney.com', + firstName: 'Mickey test', + lastName: 'VarChange', + createdAt: '2020-01-23T08:54:02.362Z', + nationality: 'USA', + random: 'superProp' + }, + page: { + path: '/destinations/mixpanel', + referrer: '', + search: '', + title: '', + url: 'https://docs.rudderstack.com/destinations/mixpanel', + category: 'destination', + initial_referrer: 'https://docs.rudderstack.com', + initial_referring_domain: 'docs.rudderstack.com' + }, + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36' + }, + integrations: { + All: true + }, + page: { + path: '/destinations/mixpanel', + referrer: '', + search: '', + title: '', + url: 'https://docs.rudderstack.com/destinations/mixpanel', + category: 'destination', + initial_referrer: 'https://docs.rudderstack.com', + initial_referring_domain: 'docs.rudderstack.com' + }, + request_ip: '[::1]:53709', + type: 'identify', + userId: 'Santiy' + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + body: { + FORM: {}, + JSON: {}, + JSON_ARRAY: { + batch: "[{\"$set_once\":{\"nationality\":\"USA\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\"}]", + }, + XML: {}, + }, + endpoint: "https://api.mixpanel.com/engage/", + files: {}, + headers: {}, + method: 'POST', + params: {}, + type: 'REST', + userId: 'Santiy', + version: '1', + }, + statusCode: 200, + }, + { + output: { + body: { + FORM: {}, + JSON: {}, + JSON_ARRAY: { + batch: "[{\"$set\":{\"$created\":\"2020-01-23T08:54:02.362Z\",\"$email\":\"TestSanity@disney.com\",\"$country_code\":\"USA\",\"$city\":\"Disney\",\"$initial_referrer\":\"https://docs.rudderstack.com\",\"$initial_referring_domain\":\"docs.rudderstack.com\",\"$name\":\"Mickey test VarChange\",\"random\":\"superProp\",\"$firstName\":\"Mickey test\",\"$lastName\":\"VarChange\",\"$browser\":\"Chrome\",\"$browser_version\":\"79.0.3945.117\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\",\"$ip\":\"0.0.0.0\",\"$time\":null}]", + }, + XML: {}, + }, + endpoint: 'https://api.mixpanel.com/engage/', + files: {}, + headers: {}, + method: 'POST', + params: {}, + type: 'REST', + userId: 'Santiy', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + } ]; From 33927d422b2aa7d64aa5c7ab3867496639e7c4b0 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 13 Nov 2023 17:18:18 +0530 Subject: [PATCH 04/16] fix: edits for supporting property paths --- .../mp/data/MPIdentifyConfig.json | 53 ++++++++++++------- src/v0/destinations/mp/transform.js | 35 +++++++----- test/integrations/destinations/mp/common.ts | 6 +++ .../destinations/mp/processor/data.ts | 10 ++-- 4 files changed, 68 insertions(+), 36 deletions(-) diff --git a/src/v0/destinations/mp/data/MPIdentifyConfig.json b/src/v0/destinations/mp/data/MPIdentifyConfig.json index 44c99fb56d..0abe7826f0 100644 --- a/src/v0/destinations/mp/data/MPIdentifyConfig.json +++ b/src/v0/destinations/mp/data/MPIdentifyConfig.json @@ -1,15 +1,13 @@ [ { "destKey": "$created", - "sourceKeys": "createdAtOnly", - "required": false, - "sourceFromGenericMap": true + "sourceKeys": ["createdAt","traits.createdAt", "context.traits.createdAt"], + "required": false }, { "destKey": "$email", - "sourceKeys": "email", - "required": false, - "sourceFromGenericMap": true + "sourceKeys": ["email","traits.email", "context.traits.email", "properties.email", "context.externalId.0.id"], + "required": false }, { "destKey": "$first_name", @@ -24,24 +22,33 @@ "context.traits.firstname", "context.traits.first_name" ], - "required": false, - "sourceFromGenericMap": true + "required": false }, { "destKey": "$last_name", - "sourceKeys": "lastName", - "required": false, - "sourceFromGenericMap": true + "sourceKeys": [ + "lastName", + "lastname", + "last_name", + "traits.lastName", + "traits.lastname", + "traits.last_name", + "context.traits.lastName", + "context.traits.lastname", + "context.traits.last_name" + ], + "required": false }, { "destKey": "$name", - "sourceKeys": "name", - "required": false, - "sourceFromGenericMap": true + "sourceKeys": ["name","traits.name", "context.traits.name"], + "required": false }, { "destKey": "$username", "sourceKeys": [ + "username", + "userName", "traits.username", "context.traits.username", "traits.userName", @@ -51,18 +58,19 @@ }, { "destKey": "$phone", - "sourceKeys": "phone", - "required": false, - "sourceFromGenericMap": true + "sourceKeys": ["phone","traits.phone", "context.traits.phone"], + "required": false }, { "destKey": "$avatar", - "sourceKeys": ["traits.avatar", "context.traits.avatar"], + "sourceKeys": ["avatar","traits.avatar", "context.traits.avatar"], "required": false }, { "destKey": "$country_code", "sourceKeys": [ + "country", + "address.country", "traits.address.country", "context.traits.address.country", "traits.country", @@ -74,6 +82,8 @@ { "destKey": "$city", "sourceKeys": [ + "city", + "address.city", "traits.city", "context.traits.city", "traits.address.city", @@ -85,6 +95,9 @@ { "destKey": "$region", "sourceKeys": [ + "state", + "address.state", + "location.region", "traits.state", "context.traits.state", "traits.address.state", @@ -95,7 +108,7 @@ }, { "destKey": "$geo_source", - "sourceKeys": "context.location.geoSource", + "sourceKeys": ["location.geoSource","context.location.geoSource"], "required": false }, { @@ -145,7 +158,7 @@ }, { "destKey": "$unsubscribed", - "sourceKeys": ["traits.unsubscribed", "context.traits.unsubscribed", "properties.unsubscribed"], + "sourceKeys": ["unsubscribed","traits.unsubscribed", "context.traits.unsubscribed", "properties.unsubscribed"], "required": false }, { diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 9cf43b88f7..69f5d2bc7f 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -17,6 +17,7 @@ const { checkInvalidRtTfEvents, handleRtTfSingleEventError, groupEventsByType, + extractCustomFields, } = require('../../util'); const { ConfigCategory, @@ -27,6 +28,7 @@ const { TRACK_MAX_BATCH_SIZE, ENGAGE_MAX_BATCH_SIZE, GROUPS_MAX_BATCH_SIZE, + MP_IDENTIFY_EXCLUSION_LIST, } = require('./config'); const { createIdentifyResponse, @@ -241,28 +243,37 @@ function trimTraits(traits, contextTraits, setOnceProperties) { const contextTraitsCopy = { ...contextTraits }; // Initialize setOnce object - const setOnce = {}; + const setOnceEligible = {}; - // Iterate over setOnceProperties and move corresponding properties to setOnce - setOnceProperties.forEach((property) => { - if (traitsCopy.hasOwnProperty(property)) { - setOnce[property] = traitsCopy[property]; - delete traitsCopy[property]; + // Step 1: find the k-v pairs of setOnceProperties in traits and contextTraits + + setOnceProperties.forEach((propertyPath) => { + const pathSegments = propertyPath.split('.'); + const propName = pathSegments[pathSegments.length - 1]; + + if (Object.keys(traitsCopy).length > 0 && get(traitsCopy, propertyPath)) { + setOnceEligible[propName] = get(traitsCopy, propertyPath); + lodash.unset(traitsCopy, propertyPath); } - if (contextTraitsCopy.hasOwnProperty(property)) { - if(!setOnce.hasOwnProperty(property)) { - setOnce[property] = contextTraitsCopy[property]; + if (Object.keys(contextTraits).length > 0 && get(contextTraitsCopy, propertyPath)) { + if(!setOnceEligible.hasOwnProperty(propName)) { + setOnceEligible[propName] = get(contextTraitsCopy, propertyPath); } - delete contextTraitsCopy[property]; + lodash.unset(contextTraitsCopy, propertyPath); } }); + // Step 2: transform properties eligible as per rudderstack declared identify event mapping + // setOnce should have all traits from message.traits and message.context.traits by now + let sentOnceTransform = constructPayload(setOnceEligible, mPIdentifyConfigJson); + + // Step 3: combine the transformed and custom setOnce traits - const sentOnceTransform = constructPayload(setOnce, mPIdentifyConfigJson); + sentOnceTransform = extractCustomFields(setOnceEligible, sentOnceTransform, 'root', MP_IDENTIFY_EXCLUSION_LIST); return { traits: traitsCopy, contextTraits: contextTraitsCopy, - sentOnceTransform, + setOnce: sentOnceTransform, }; } diff --git a/test/integrations/destinations/mp/common.ts b/test/integrations/destinations/mp/common.ts index e2daaa8622..f5efc4cc48 100644 --- a/test/integrations/destinations/mp/common.ts +++ b/test/integrations/destinations/mp/common.ts @@ -38,6 +38,12 @@ const destinationWithSetOnceProperty = { setOnceProperties: [ { property: 'nationality' + }, + { + property: 'firstName' + }, + { + property: 'address.city' } ], superProperties: [ diff --git a/test/integrations/destinations/mp/processor/data.ts b/test/integrations/destinations/mp/processor/data.ts index 7038a24fac..e5313a93fe 100644 --- a/test/integrations/destinations/mp/processor/data.ts +++ b/test/integrations/destinations/mp/processor/data.ts @@ -5992,7 +5992,7 @@ export const data = [ request: { body: [ { - description: 'Alias: with property beyond exclusion list', + description: 'Alias: with property beyond and within exclusion list', destination: destinationWithSetOnceProperty, message: { anonymousId: 'e6ab2c5e-2cda-44a9-a962-e2f67df78bca', @@ -6018,7 +6018,9 @@ export const data = [ density: 2 }, traits: { - city: 'Disney', + address: { + city: 'Disney', + }, country: 'USA', email: 'TestSanity@disney.com', firstName: 'Mickey test', @@ -6072,7 +6074,7 @@ export const data = [ FORM: {}, JSON: {}, JSON_ARRAY: { - batch: "[{\"$set_once\":{\"nationality\":\"USA\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\"}]", + batch: "[{\"$set_once\":{\"$first_name\":\"Mickey test\",\"$city\":\"Disney\",\"nationality\":\"USA\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\"}]", }, XML: {}, }, @@ -6093,7 +6095,7 @@ export const data = [ FORM: {}, JSON: {}, JSON_ARRAY: { - batch: "[{\"$set\":{\"$created\":\"2020-01-23T08:54:02.362Z\",\"$email\":\"TestSanity@disney.com\",\"$country_code\":\"USA\",\"$city\":\"Disney\",\"$initial_referrer\":\"https://docs.rudderstack.com\",\"$initial_referring_domain\":\"docs.rudderstack.com\",\"$name\":\"Mickey test VarChange\",\"random\":\"superProp\",\"$firstName\":\"Mickey test\",\"$lastName\":\"VarChange\",\"$browser\":\"Chrome\",\"$browser_version\":\"79.0.3945.117\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\",\"$ip\":\"0.0.0.0\",\"$time\":null}]", + "batch": "[{\"$set\":{\"$created\":\"2020-01-23T08:54:02.362Z\",\"$email\":\"TestSanity@disney.com\",\"$country_code\":\"USA\",\"$initial_referrer\":\"https://docs.rudderstack.com\",\"$initial_referring_domain\":\"docs.rudderstack.com\",\"random\":\"superProp\",\"$lastName\":\"VarChange\",\"$browser\":\"Chrome\",\"$browser_version\":\"79.0.3945.117\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\",\"$ip\":\"0.0.0.0\",\"$time\":null}]" }, XML: {}, }, From 5f3a35caca93fe249a7f33fcf2caddf654f9e96c Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 13 Nov 2023 18:06:05 +0530 Subject: [PATCH 05/16] fix: delete wrong test case --- .../destinations/mp/processor/data.ts | 132 ------------------ 1 file changed, 132 deletions(-) diff --git a/test/integrations/destinations/mp/processor/data.ts b/test/integrations/destinations/mp/processor/data.ts index e5313a93fe..e36bda3be4 100644 --- a/test/integrations/destinations/mp/processor/data.ts +++ b/test/integrations/destinations/mp/processor/data.ts @@ -5981,137 +5981,5 @@ export const data = [ ], }, }, - }, - { - name: 'mp', - description: 'Test Set Once Property', - feature: 'processor', - module: 'destination', - version: 'v0', - input: { - request: { - body: [ - { - description: 'Alias: with property beyond and within exclusion list', - destination: destinationWithSetOnceProperty, - message: { - anonymousId: 'e6ab2c5e-2cda-44a9-a962-e2f67df78bca', - channel: 'web', - context: { - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.0.5' - }, - ip: '0.0.0.0', - library: { - name: 'RudderLabs JavaScript SDK', - version: '1.0.5' - }, - locale: 'en-GB', - os: { - name: '', - version: '' - }, - screen: { - density: 2 - }, - traits: { - address: { - city: 'Disney', - }, - country: 'USA', - email: 'TestSanity@disney.com', - firstName: 'Mickey test', - lastName: 'VarChange', - createdAt: '2020-01-23T08:54:02.362Z', - nationality: 'USA', - random: 'superProp' - }, - page: { - path: '/destinations/mixpanel', - referrer: '', - search: '', - title: '', - url: 'https://docs.rudderstack.com/destinations/mixpanel', - category: 'destination', - initial_referrer: 'https://docs.rudderstack.com', - initial_referring_domain: 'docs.rudderstack.com' - }, - userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36' - }, - integrations: { - All: true - }, - page: { - path: '/destinations/mixpanel', - referrer: '', - search: '', - title: '', - url: 'https://docs.rudderstack.com/destinations/mixpanel', - category: 'destination', - initial_referrer: 'https://docs.rudderstack.com', - initial_referring_domain: 'docs.rudderstack.com' - }, - request_ip: '[::1]:53709', - type: 'identify', - userId: 'Santiy' - }, - }, - ], - method: 'POST', - }, - pathSuffix: '', - }, - output: { - response: { - status: 200, - body: [ - { - output: { - body: { - FORM: {}, - JSON: {}, - JSON_ARRAY: { - batch: "[{\"$set_once\":{\"$first_name\":\"Mickey test\",\"$city\":\"Disney\",\"nationality\":\"USA\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\"}]", - }, - XML: {}, - }, - endpoint: "https://api.mixpanel.com/engage/", - files: {}, - headers: {}, - method: 'POST', - params: {}, - type: 'REST', - userId: 'Santiy', - version: '1', - }, - statusCode: 200, - }, - { - output: { - body: { - FORM: {}, - JSON: {}, - JSON_ARRAY: { - "batch": "[{\"$set\":{\"$created\":\"2020-01-23T08:54:02.362Z\",\"$email\":\"TestSanity@disney.com\",\"$country_code\":\"USA\",\"$initial_referrer\":\"https://docs.rudderstack.com\",\"$initial_referring_domain\":\"docs.rudderstack.com\",\"random\":\"superProp\",\"$lastName\":\"VarChange\",\"$browser\":\"Chrome\",\"$browser_version\":\"79.0.3945.117\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\",\"$ip\":\"0.0.0.0\",\"$time\":null}]" - }, - XML: {}, - }, - endpoint: 'https://api.mixpanel.com/engage/', - files: {}, - headers: {}, - method: 'POST', - params: {}, - type: 'REST', - userId: 'Santiy', - version: '1', - }, - statusCode: 200, - }, - ], - }, - }, } ]; From ea60161facfcea69f5a32c8bec840bc518b14ff0 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 13 Nov 2023 18:22:31 +0530 Subject: [PATCH 06/16] fix: test cases --- test/integrations/destinations/mp/common.ts | 36 +++++------ .../destinations/mp/processor/data.ts | 63 ++++++++++--------- 2 files changed, 49 insertions(+), 50 deletions(-) diff --git a/test/integrations/destinations/mp/common.ts b/test/integrations/destinations/mp/common.ts index f5efc4cc48..76ed25a760 100644 --- a/test/integrations/destinations/mp/common.ts +++ b/test/integrations/destinations/mp/common.ts @@ -21,47 +21,41 @@ const sampleDestination = { }; const destinationWithSetOnceProperty = { - ID: '2Xz4szkZeBMJRgO5TfvLclsxkxz', - Name: 'Mixpanel Dev', - DestinationDefinition: { - ID: '2WcAA63MqbQVSIrgPR0mXuxF4ut', - Name: 'MP', - DisplayName: 'Mixpanel', - ResponseRules: {} - }, Config: { apiSecret: 'dummySecret', dataResidency: 'us', - eventDelivery: false, - eventDeliveryTS: 1699848284554, identityMergeApi: 'simplified', setOnceProperties: [ { - property: 'nationality' + property: 'nationality', }, { - property: 'firstName' + property: 'firstName', }, { - property: 'address.city' - } + property: 'address.city', + }, ], superProperties: [ { - property: 'random' - } + property: 'random', + }, ], token: 'dummyToken', useNativeSDK: false, useNewMapping: false, userDeletionApi: 'engage', - whitelistedEvents: [] + whitelistedEvents: [], + }, + DestinationDefinition: { + DisplayName: 'Kiss Metrics', + ID: '1WhbSZ6uA3H5ChVifHpfL2H6sie', + Name: 'MIXPANEL', }, Enabled: true, - WorkspaceID: 123345, + ID: '1WhcOCGgj9asZu850HvugU2C3Aq', + Name: 'Kiss Metrics', Transformations: [], - IsProcessorEnabled: true, - RevisionID: 12345 -} +}; export { sampleDestination, defaultMockFns, destinationWithSetOnceProperty }; diff --git a/test/integrations/destinations/mp/processor/data.ts b/test/integrations/destinations/mp/processor/data.ts index e36bda3be4..30da96ff9f 100644 --- a/test/integrations/destinations/mp/processor/data.ts +++ b/test/integrations/destinations/mp/processor/data.ts @@ -5862,7 +5862,7 @@ export const data = [ request: { body: [ { - description: 'Alias: with property beyond exclusion list', + description: 'Alias: with property beyond and within exclusion list', destination: destinationWithSetOnceProperty, message: { anonymousId: 'e6ab2c5e-2cda-44a9-a962-e2f67df78bca', @@ -5872,30 +5872,32 @@ export const data = [ build: '1.0.0', name: 'RudderLabs JavaScript SDK', namespace: 'com.rudderlabs.javascript', - version: '1.0.5' + version: '1.0.5', }, ip: '0.0.0.0', library: { name: 'RudderLabs JavaScript SDK', - version: '1.0.5' + version: '1.0.5', }, locale: 'en-GB', os: { name: '', - version: '' + version: '', }, screen: { - density: 2 + density: 2, }, traits: { - city: 'Disney', + address: { + city: 'Disney', + }, country: 'USA', email: 'TestSanity@disney.com', firstName: 'Mickey test', lastName: 'VarChange', createdAt: '2020-01-23T08:54:02.362Z', nationality: 'USA', - random: 'superProp' + random: 'superProp', }, page: { path: '/destinations/mixpanel', @@ -5905,12 +5907,13 @@ export const data = [ url: 'https://docs.rudderstack.com/destinations/mixpanel', category: 'destination', initial_referrer: 'https://docs.rudderstack.com', - initial_referring_domain: 'docs.rudderstack.com' + initial_referring_domain: 'docs.rudderstack.com', }, - userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36' + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', }, integrations: { - All: true + All: true, }, page: { path: '/destinations/mixpanel', @@ -5920,11 +5923,11 @@ export const data = [ url: 'https://docs.rudderstack.com/destinations/mixpanel', category: 'destination', initial_referrer: 'https://docs.rudderstack.com', - initial_referring_domain: 'docs.rudderstack.com' + initial_referring_domain: 'docs.rudderstack.com', }, request_ip: '[::1]:53709', type: 'identify', - userId: 'Santiy' + userId: 'Santiy', }, }, ], @@ -5938,48 +5941,50 @@ export const data = [ body: [ { output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.mixpanel.com/engage/', + headers: {}, + params: {}, body: { - FORM: {}, JSON: {}, JSON_ARRAY: { - batch: "[{\"$set_once\":{\"nationality\":\"USA\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\"}]", + batch: + '[{"$set_once":{"$first_name":"Mickey test","$city":"Disney","nationality":"USA"},"$token":"dummyToken","$distinct_id":"Santiy"}]', }, XML: {}, + FORM: {}, }, - endpoint: "https://api.mixpanel.com/engage/", files: {}, - headers: {}, - method: 'POST', - params: {}, - type: 'REST', userId: 'Santiy', - version: '1', }, statusCode: 200, }, { output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.mixpanel.com/engage/', + headers: {}, + params: {}, body: { - FORM: {}, JSON: {}, JSON_ARRAY: { - batch: "[{\"$set\":{\"$created\":\"2020-01-23T08:54:02.362Z\",\"$email\":\"TestSanity@disney.com\",\"$country_code\":\"USA\",\"$city\":\"Disney\",\"$initial_referrer\":\"https://docs.rudderstack.com\",\"$initial_referring_domain\":\"docs.rudderstack.com\",\"$name\":\"Mickey test VarChange\",\"random\":\"superProp\",\"$firstName\":\"Mickey test\",\"$lastName\":\"VarChange\",\"$browser\":\"Chrome\",\"$browser_version\":\"79.0.3945.117\"},\"$token\":\"dummyToken\",\"$distinct_id\":\"Santiy\",\"$ip\":\"0.0.0.0\",\"$time\":null}]", + batch: + '[{"$set":{"$created":"2020-01-23T08:54:02.362Z","$email":"TestSanity@disney.com","$country_code":"USA","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","random":"superProp","$lastName":"VarChange","$browser":"Chrome","$browser_version":"79.0.3945.117"},"$token":"dummyToken","$distinct_id":"Santiy","$ip":"0.0.0.0","$time":null}]', }, XML: {}, + FORM: {}, }, - endpoint: 'https://api.mixpanel.com/engage/', files: {}, - headers: {}, - method: 'POST', - params: {}, - type: 'REST', userId: 'Santiy', - version: '1', }, statusCode: 200, }, ], }, }, - } + }, ]; From 1dafad7bbfa9f2d059b7d3594c73edca17a22a7f Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Tue, 14 Nov 2023 18:08:04 +0530 Subject: [PATCH 07/16] fix: removed unnecessary code --- src/v0/destinations/mp/transform.js | 18 ++++++++---------- src/v0/destinations/mp/util.js | 13 ++++--------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 69f5d2bc7f..7004bc6e94 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -110,13 +110,6 @@ const responseBuilderSimple = (payload, message, eventType, destConfig) => { strict: credentials.params.strict, }; break; - // case 'setOnce': - // response.endpoint = dataResidency === 'eu' ? `${BASE_ENDPOINT_EU}/engage#profile-set-once` : `${BASE_ENDPOINT}/engage#profile-set-once`; - // response.headers = { - // 'Content-Type': 'application/json', - // 'accept': 'text/plain', - // }; - // break; default: response.endpoint = dataResidency === 'eu' ? `${BASE_ENDPOINT_EU}/engage/` : `${BASE_ENDPOINT}/engage/`; @@ -256,7 +249,7 @@ function trimTraits(traits, contextTraits, setOnceProperties) { lodash.unset(traitsCopy, propertyPath); } if (Object.keys(contextTraits).length > 0 && get(contextTraitsCopy, propertyPath)) { - if(!setOnceEligible.hasOwnProperty(propName)) { + if (!setOnceEligible.hasOwnProperty(propName)) { setOnceEligible[propName] = get(contextTraitsCopy, propertyPath); } lodash.unset(contextTraitsCopy, propertyPath); @@ -268,7 +261,12 @@ function trimTraits(traits, contextTraits, setOnceProperties) { // Step 3: combine the transformed and custom setOnce traits - sentOnceTransform = extractCustomFields(setOnceEligible, sentOnceTransform, 'root', MP_IDENTIFY_EXCLUSION_LIST); + sentOnceTransform = extractCustomFields( + setOnceEligible, + sentOnceTransform, + 'root', + MP_IDENTIFY_EXCLUSION_LIST, + ); return { traits: traitsCopy, @@ -312,7 +310,7 @@ const processIdentifyEvents = async (message, type, destination) => { // Creating the user profile // https://developer.mixpanel.com/reference/profile-set - returnValue.push(createIdentifyResponse(messageClone, type, destination, responseBuilderSimple, setOnceProperties)); + returnValue.push(createIdentifyResponse(messageClone, type, destination, responseBuilderSimple)); if ( destination.Config?.identityMergeApi !== 'simplified' && diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index dd4bbccc93..4acc7824fc 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -34,7 +34,7 @@ const mPProfileIosConfigJson = mappingConfig[ConfigCategory.PROFILE_IOS.name]; * @param {*} useNewMapping a variable to support backward compatibility * @returns */ -const getTransformedJSON = (message, mappingJson, useNewMapping, setOnceProperties) => { +const getTransformedJSON = (message, mappingJson, useNewMapping) => { let rawPayload = constructPayload(message, mappingJson); if ( isDefined(rawPayload.$geo_source) && @@ -47,16 +47,11 @@ const getTransformedJSON = (message, mappingJson, useNewMapping, setOnceProperti set(rawPayload, '$name', getFullName(message)); } - let newExclusionList = [...MP_IDENTIFY_EXCLUSION_LIST]; - if (setOnceProperties.length > 0) { - newExclusionList = [...newExclusionList, ...setOnceProperties]; - } - rawPayload = extractCustomFields( message, rawPayload, ['traits', 'context.traits'], - newExclusionList, + MP_IDENTIFY_EXCLUSION_LIST, ); /* @@ -104,11 +99,11 @@ const getTransformedJSON = (message, mappingJson, useNewMapping, setOnceProperti * @param {*} responseBuilderSimple function to generate response * @returns */ -const createIdentifyResponse = (message, type, destination, responseBuilderSimple, setOnceProperties) => { +const createIdentifyResponse = (message, type, destination, responseBuilderSimple) => { // this variable is used for supporting backward compatibility const { useNewMapping, token } = destination.Config; // user payload created - const properties = getTransformedJSON(message, mPIdentifyConfigJson, useNewMapping, setOnceProperties); + const properties = getTransformedJSON(message, mPIdentifyConfigJson, useNewMapping); const payload = { $set: properties, From 3273c622ef22624a2870287a97da5f03a5ea0f5c Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Thu, 16 Nov 2023 13:45:05 +0530 Subject: [PATCH 08/16] fix: adding unit test cases for trimTraits --- src/v0/destinations/mp/transform.js | 49 +------------ src/v0/destinations/mp/util.js | 64 +++++++++++++++++ src/v0/destinations/mp/util.test.js | 105 ++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 48 deletions(-) diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 7004bc6e94..ac1209194e 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -17,7 +17,6 @@ const { checkInvalidRtTfEvents, handleRtTfSingleEventError, groupEventsByType, - extractCustomFields, } = require('../../util'); const { ConfigCategory, @@ -28,7 +27,6 @@ const { TRACK_MAX_BATCH_SIZE, ENGAGE_MAX_BATCH_SIZE, GROUPS_MAX_BATCH_SIZE, - MP_IDENTIFY_EXCLUSION_LIST, } = require('./config'); const { createIdentifyResponse, @@ -38,12 +36,12 @@ const { groupEventsByEndpoint, batchEvents, parseConfigArray, + trimTraits, } = require('./util'); const { CommonUtils } = require('../../../util/common'); // ref: https://help.mixpanel.com/hc/en-us/articles/115004613766-Default-Properties-Collected-by-Mixpanel const mPEventPropertiesConfigJson = mappingConfig[ConfigCategory.EVENT_PROPERTIES.name]; -const mPIdentifyConfigJson = mappingConfig[ConfigCategory.IDENTIFY.name]; const setImportCredentials = (destConfig) => { const endpoint = @@ -230,51 +228,6 @@ const processTrack = (message, destination) => { return returnValue; }; -function trimTraits(traits, contextTraits, setOnceProperties) { - // Create a copy of the original traits object - const traitsCopy = { ...traits }; - const contextTraitsCopy = { ...contextTraits }; - - // Initialize setOnce object - const setOnceEligible = {}; - - // Step 1: find the k-v pairs of setOnceProperties in traits and contextTraits - - setOnceProperties.forEach((propertyPath) => { - const pathSegments = propertyPath.split('.'); - const propName = pathSegments[pathSegments.length - 1]; - - if (Object.keys(traitsCopy).length > 0 && get(traitsCopy, propertyPath)) { - setOnceEligible[propName] = get(traitsCopy, propertyPath); - lodash.unset(traitsCopy, propertyPath); - } - if (Object.keys(contextTraits).length > 0 && get(contextTraitsCopy, propertyPath)) { - if (!setOnceEligible.hasOwnProperty(propName)) { - setOnceEligible[propName] = get(contextTraitsCopy, propertyPath); - } - lodash.unset(contextTraitsCopy, propertyPath); - } - }); - // Step 2: transform properties eligible as per rudderstack declared identify event mapping - // setOnce should have all traits from message.traits and message.context.traits by now - let sentOnceTransform = constructPayload(setOnceEligible, mPIdentifyConfigJson); - - // Step 3: combine the transformed and custom setOnce traits - - sentOnceTransform = extractCustomFields( - setOnceEligible, - sentOnceTransform, - 'root', - MP_IDENTIFY_EXCLUSION_LIST, - ); - - return { - traits: traitsCopy, - contextTraits: contextTraitsCopy, - setOnce: sentOnceTransform, - }; -} - const createSetOnceResponse = (message, type, destination, setOnce) => { const payload = { $set_once: setOnce, diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index 4acc7824fc..de6ff7773a 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -1,3 +1,4 @@ +const lodash = require('lodash'); const set = require('set-value'); const get = require('get-value'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); @@ -329,6 +330,68 @@ const parseConfigArray = (arr, key) => { return arr.map((item) => item[key]); }; +/** + * Trims the traits and contextTraits objects based on the setOnceProperties array and returns an object containing the modified traits, contextTraits, and setOnce properties. + * + * @param {object} traits - An object representing the traits. + * @param {object} contextTraits - An object representing the context traits. + * @param {string[]} setOnceProperties - An array of property paths to be considered for the setOnce transformation. + * @returns {object} - An object containing the modified traits, contextTraits, and setOnce properties. + * + * @example + * const traits = { name: 'John', age: 30 }; + * const contextTraits = { country: 'USA', language: 'English', address: { city: 'New York', state: 'NY' }}}; + * const setOnceProperties = ['name', 'country', 'address.city']; + * + * const result = trimTraits(traits, contextTraits, setOnceProperties); + * console.log(result); + * // Output: { traits: { age: 30 }, contextTraits: { language: 'English' }, setOnce: { $name: 'John', $country_code: 'USA', city: 'New York'} } + */ +function trimTraits(traits, contextTraits, setOnceProperties) { + // Create a copy of the original traits object + const traitsCopy = { ...traits }; + const contextTraitsCopy = { ...contextTraits }; + + // Initialize setOnce object + const setOnceEligible = {}; + + // Step 1: find the k-v pairs of setOnceProperties in traits and contextTraits + + setOnceProperties.forEach((propertyPath) => { + const pathSegments = propertyPath.split('.'); + const propName = pathSegments[pathSegments.length - 1]; + + if (Object.keys(traitsCopy).length > 0 && get(traitsCopy, propertyPath)) { + setOnceEligible[propName] = get(traitsCopy, propertyPath); + lodash.unset(traitsCopy, propertyPath); + } + if (Object.keys(contextTraits).length > 0 && get(contextTraitsCopy, propertyPath)) { + if (!setOnceEligible.hasOwnProperty(propName)) { + setOnceEligible[propName] = get(contextTraitsCopy, propertyPath); + } + lodash.unset(contextTraitsCopy, propertyPath); + } + }); + // Step 2: transform properties eligible as per rudderstack declared identify event mapping + // setOnce should have all traits from message.traits and message.context.traits by now + let sentOnceTransform = constructPayload(setOnceEligible, mPIdentifyConfigJson); + + // Step 3: combine the transformed and custom setOnce traits + + sentOnceTransform = extractCustomFields( + setOnceEligible, + sentOnceTransform, + 'root', + MP_IDENTIFY_EXCLUSION_LIST, + ); + + return { + traits: traitsCopy, + contextTraits: contextTraitsCopy, + setOnce: sentOnceTransform, + }; +} + module.exports = { createIdentifyResponse, isImportAuthCredentialsAvailable, @@ -338,4 +401,5 @@ module.exports = { batchEvents, combineBatchRequestsWithSameJobIds, parseConfigArray, + trimTraits, }; diff --git a/src/v0/destinations/mp/util.test.js b/src/v0/destinations/mp/util.test.js index 6d5b24766d..f90433561d 100644 --- a/src/v0/destinations/mp/util.test.js +++ b/src/v0/destinations/mp/util.test.js @@ -4,6 +4,7 @@ const { batchEvents, generateBatchedPayloadForArray, buildUtmParams, + trimTraits, } = require('./util'); const { FEATURE_GZIP_SUPPORT } = require('../../util/constant'); @@ -602,4 +603,108 @@ describe('Mixpanel utils test', () => { }); }); }); + describe('Unit test cases for trimTraits', () => { + // Given a valid traits object and contextTraits object, and a valid setOnceProperties array, the function should return an object containing traits, contextTraits, and setOnce properties. + it('should return an object containing traits, contextTraits, and setOnce properties when given valid inputs', () => { + const traits = { name: 'John', age: 30 }; + const contextTraits = { email: 'john@example.com' }; + const setOnceProperties = ['name', 'email']; + + const result = trimTraits(traits, contextTraits, setOnceProperties); + console.log(result); + + expect(result).toEqual({ + traits: { + age: 30, + }, + contextTraits: {}, + setOnce: { $name: 'John', $email: 'john@example.com' }, + }); + }); + + // Given an empty traits object and contextTraits object, and a valid setOnceProperties array, the function should return an object containing empty traits and contextTraits, and an empty setOnce property. + it('should return an object containing empty traits and contextTraits, and an empty setOnce property when given empty traits and contextTraits objects', () => { + const traits = {}; + const contextTraits = {}; + const setOnceProperties = ['name', 'email']; + + const result = trimTraits(traits, contextTraits, setOnceProperties); + + expect(result).toEqual({ + traits: {}, + contextTraits: {}, + setOnce: {}, + }); + }); + + // Given an empty setOnceProperties array, the function should return an object containing the original traits and contextTraits objects, and an empty setOnce property. + it('should return an object containing the original traits and contextTraits objects, and an empty setOnce property when given an empty setOnceProperties array', () => { + const traits = { name: 'John', age: 30 }; + const contextTraits = { email: 'john@example.com' }; + const setOnceProperties = []; + + const result = trimTraits(traits, contextTraits, setOnceProperties); + + expect(result).toEqual({ + traits: { name: 'John', age: 30 }, + contextTraits: { email: 'john@example.com' }, + setOnce: {}, + }); + }); + + // Given a null traits object and contextTraits object, and a valid setOnceProperties array, the function should throw an error. + it('should throw an error when given a null traits object and contextTraits object', () => { + const traits = null; + const contextTraits = null; + const setOnceProperties = ['name', 'email']; + + expect(() => { + trimTraits(traits, contextTraits, setOnceProperties); + }).toThrow(); + }); + + // Given a setOnceProperties array containing properties that do not exist in either traits or contextTraits objects, the function should not add the property to the setOnce property. + it('should not add properties to the setOnce property when given setOnceProperties array with non-existent properties', () => { + const traits = { name: 'John', age: 30 }; + const contextTraits = { email: 'john@example.com' }; + const setOnceProperties = ['name', 'email', 'address']; + + const result = trimTraits(traits, contextTraits, setOnceProperties); + + expect(result).toEqual({ + traits: { age: 30 }, + contextTraits: {}, + setOnce: { $name: 'John', $email: 'john@example.com' }, + }); + }); + + // Given a setOnceProperties array containing properties with nested paths that do not exist in either traits or contextTraits objects, the function should not add the property to the setOnce property. + it('should not add properties to the setOnce property when given setOnceProperties array with non-existent nested properties', () => { + const traits = { name: 'John', age: 30, address: 'kolkata' }; + const contextTraits = { email: 'john@example.com' }; + const setOnceProperties = ['name', 'email', 'address.city']; + + const result = trimTraits(traits, contextTraits, setOnceProperties); + + expect(result).toEqual({ + traits: { age: 30, address: 'kolkata' }, + contextTraits: {}, + setOnce: { $name: 'John', $email: 'john@example.com' }, + }); + }); + + it('should add properties to the setOnce property when given setOnceProperties array with existent nested properties', () => { + const traits = { name: 'John', age: 30, address: { city: 'kolkata' } }; + const contextTraits = { email: 'john@example.com' }; + const setOnceProperties = ['name', 'email', 'address.city']; + + const result = trimTraits(traits, contextTraits, setOnceProperties); + + expect(result).toEqual({ + traits: { age: 30, address: {} }, + contextTraits: {}, + setOnce: { $name: 'John', $email: 'john@example.com', $city: 'kolkata' }, + }); + }); + }); }); From 9271f111892746778a697546b394ca5e255f8b20 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Thu, 16 Nov 2023 15:12:39 +0530 Subject: [PATCH 09/16] fix: changing the order of priority in property mapping --- .../mp/data/MPIdentifyConfig.json | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/src/v0/destinations/mp/data/MPIdentifyConfig.json b/src/v0/destinations/mp/data/MPIdentifyConfig.json index 0abe7826f0..6934735f67 100644 --- a/src/v0/destinations/mp/data/MPIdentifyConfig.json +++ b/src/v0/destinations/mp/data/MPIdentifyConfig.json @@ -1,114 +1,120 @@ [ { "destKey": "$created", - "sourceKeys": ["createdAt","traits.createdAt", "context.traits.createdAt"], + "sourceKeys": ["traits.createdAt", "context.traits.createdAt", "createdAt"], "required": false }, { "destKey": "$email", - "sourceKeys": ["email","traits.email", "context.traits.email", "properties.email", "context.externalId.0.id"], + "sourceKeys": [ + "traits.email", + "context.traits.email", + "properties.email", + "context.externalId.0.id", + "email" + ], "required": false }, { "destKey": "$first_name", "sourceKeys": [ - "firstName", - "firstname", - "first_name", "traits.firstName", "traits.firstname", "traits.first_name", "context.traits.firstName", "context.traits.firstname", - "context.traits.first_name" + "context.traits.first_name", + "firstName", + "firstname", + "first_name" ], "required": false }, { "destKey": "$last_name", "sourceKeys": [ - "lastName", - "lastname", - "last_name", "traits.lastName", "traits.lastname", "traits.last_name", "context.traits.lastName", "context.traits.lastname", - "context.traits.last_name" + "context.traits.last_name", + "lastName", + "lastname", + "last_name" ], "required": false }, { "destKey": "$name", - "sourceKeys": ["name","traits.name", "context.traits.name"], + "sourceKeys": ["traits.name", "context.traits.name", "name"], "required": false }, { "destKey": "$username", "sourceKeys": [ - "username", - "userName", "traits.username", "context.traits.username", "traits.userName", - "context.traits.userName" + "context.traits.userName", + "username", + "userName" ], "required": false }, { "destKey": "$phone", - "sourceKeys": ["phone","traits.phone", "context.traits.phone"], + "sourceKeys": ["traits.phone", "context.traits.phone", "phone"], "required": false }, { "destKey": "$avatar", - "sourceKeys": ["avatar","traits.avatar", "context.traits.avatar"], + "sourceKeys": ["traits.avatar", "context.traits.avatar", "avatar"], "required": false }, { "destKey": "$country_code", "sourceKeys": [ - "country", - "address.country", "traits.address.country", "context.traits.address.country", "traits.country", "context.traits.country", - "context.location.country" + "context.location.country", + "country", + "address.country" ], "required": false }, { "destKey": "$city", "sourceKeys": [ - "city", - "address.city", "traits.city", "context.traits.city", "traits.address.city", "context.traits.address.city", - "context.location.city" + "context.location.city", + "city", + "address.city" ], "required": false }, { "destKey": "$region", "sourceKeys": [ - "state", - "address.state", - "location.region", "traits.state", "context.traits.state", "traits.address.state", "context.traits.address.state", - "context.location.region" + "context.location.region", + "state", + "address.state", + "location.region" ], "required": false }, { "destKey": "$geo_source", - "sourceKeys": ["location.geoSource","context.location.geoSource"], + "sourceKeys": ["location.geoSource", "context.location.geoSource"], "required": false }, { @@ -158,7 +164,12 @@ }, { "destKey": "$unsubscribed", - "sourceKeys": ["unsubscribed","traits.unsubscribed", "context.traits.unsubscribed", "properties.unsubscribed"], + "sourceKeys": [ + "traits.unsubscribed", + "context.traits.unsubscribed", + "properties.unsubscribed", + "unsubscribed" + ], "required": false }, { From c0ac6305b355f6c02426d401d48aa28c4c5a0051 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Thu, 16 Nov 2023 15:26:10 +0530 Subject: [PATCH 10/16] fix: edited distinct id logic --- src/v0/destinations/mp/transform.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index ac1209194e..9e80be68cd 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -234,6 +234,11 @@ const createSetOnceResponse = (message, type, destination, setOnce) => { $token: destination.Config.token, $distinct_id: message.userId || message.anonymousId, }; + + if (destination?.Config.identityMergeApi === 'simplified') { + payload.$distinct_id = message.userId || `$device:${message.anonymousId}`; + } + return responseBuilderSimple(payload, message, type, destination.Config); }; From 94c3539d0f6fc892c776223810a064b6614672d5 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Thu, 16 Nov 2023 15:29:48 +0530 Subject: [PATCH 11/16] fix: small edit --- src/v0/destinations/mp/transform.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 9e80be68cd..6f4bcd07bb 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -477,7 +477,6 @@ const processRouterDest = async (inputs, reqMetadata) => { destination: event.destination, }; } - // console.log('event', JSON.stringify(event)); let processedEvents = await process(event); processedEvents = CommonUtils.toArray(processedEvents); return processedEvents.map((res) => ({ From 6ea3d1e520540d176530fc8871ca8610497ac8fe Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Thu, 16 Nov 2023 16:07:26 +0530 Subject: [PATCH 12/16] fix: review comments addressed --- src/v0/destinations/mp/transform.js | 8 +++--- src/v0/destinations/mp/util.js | 42 ++++++++++++++++------------- src/v0/destinations/mp/util.test.js | 11 -------- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 6f4bcd07bb..cab50b976f 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -261,9 +261,11 @@ const processIdentifyEvents = async (message, type, destination) => { ); messageClone.traits = seggregatedTraits.traits; messageClone.context.traits = seggregatedTraits.contextTraits; - returnValue.push( - createSetOnceResponse(messageClone, type, destination, seggregatedTraits.setOnce), - ); + if (Object.keys(seggregatedTraits.setOnce).length > 0) { + returnValue.push( + createSetOnceResponse(messageClone, type, destination, seggregatedTraits.setOnce), + ); + } } // Creating the user profile diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index de6ff7773a..a2dc2f2563 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -15,6 +15,7 @@ const { defaultBatchRequestConfig, IsGzipSupported, isObject, + isDefinedAndNotNullAndNotEmpty, } = require('../../util'); const { ConfigCategory, @@ -348,6 +349,7 @@ const parseConfigArray = (arr, key) => { * // Output: { traits: { age: 30 }, contextTraits: { language: 'English' }, setOnce: { $name: 'John', $country_code: 'USA', city: 'New York'} } */ function trimTraits(traits, contextTraits, setOnceProperties) { + let sentOnceTransform; // Create a copy of the original traits object const traitsCopy = { ...traits }; const contextTraitsCopy = { ...contextTraits }; @@ -358,37 +360,41 @@ function trimTraits(traits, contextTraits, setOnceProperties) { // Step 1: find the k-v pairs of setOnceProperties in traits and contextTraits setOnceProperties.forEach((propertyPath) => { - const pathSegments = propertyPath.split('.'); - const propName = pathSegments[pathSegments.length - 1]; + const propName = lodash.last(propertyPath.split('.')); - if (Object.keys(traitsCopy).length > 0 && get(traitsCopy, propertyPath)) { - setOnceEligible[propName] = get(traitsCopy, propertyPath); + const traitsValue = get(traitsCopy, propertyPath); + const contextTraitsValue = get(contextTraitsCopy, propertyPath); + + if (isDefinedAndNotNullAndNotEmpty(traitsValue)) { + setOnceEligible[propName] = traitsValue; lodash.unset(traitsCopy, propertyPath); } - if (Object.keys(contextTraits).length > 0 && get(contextTraitsCopy, propertyPath)) { + if (isDefinedAndNotNullAndNotEmpty(contextTraitsValue)) { if (!setOnceEligible.hasOwnProperty(propName)) { - setOnceEligible[propName] = get(contextTraitsCopy, propertyPath); + setOnceEligible[propName] = contextTraitsValue; } lodash.unset(contextTraitsCopy, propertyPath); } }); - // Step 2: transform properties eligible as per rudderstack declared identify event mapping - // setOnce should have all traits from message.traits and message.context.traits by now - let sentOnceTransform = constructPayload(setOnceEligible, mPIdentifyConfigJson); - - // Step 3: combine the transformed and custom setOnce traits - sentOnceTransform = extractCustomFields( - setOnceEligible, - sentOnceTransform, - 'root', - MP_IDENTIFY_EXCLUSION_LIST, - ); + if (setOnceEligible && Object.keys(setOnceEligible).length > 0) { + // Step 2: transform properties eligible as per rudderstack declared identify event mapping + // setOnce should have all traits from message.traits and message.context.traits by now + sentOnceTransform = constructPayload(setOnceEligible, mPIdentifyConfigJson); + + // Step 3: combine the transformed and custom setOnce traits + sentOnceTransform = extractCustomFields( + setOnceEligible, + sentOnceTransform, + 'root', + MP_IDENTIFY_EXCLUSION_LIST, + ); + } return { traits: traitsCopy, contextTraits: contextTraitsCopy, - setOnce: sentOnceTransform, + setOnce: sentOnceTransform || {}, }; } diff --git a/src/v0/destinations/mp/util.test.js b/src/v0/destinations/mp/util.test.js index f90433561d..028dfa3be3 100644 --- a/src/v0/destinations/mp/util.test.js +++ b/src/v0/destinations/mp/util.test.js @@ -652,17 +652,6 @@ describe('Mixpanel utils test', () => { }); }); - // Given a null traits object and contextTraits object, and a valid setOnceProperties array, the function should throw an error. - it('should throw an error when given a null traits object and contextTraits object', () => { - const traits = null; - const contextTraits = null; - const setOnceProperties = ['name', 'email']; - - expect(() => { - trimTraits(traits, contextTraits, setOnceProperties); - }).toThrow(); - }); - // Given a setOnceProperties array containing properties that do not exist in either traits or contextTraits objects, the function should not add the property to the setOnce property. it('should not add properties to the setOnce property when given setOnceProperties array with non-existent properties', () => { const traits = { name: 'John', age: 30 }; From 91beb1ee98ace6fdccfd701609422d9e9bf86ea7 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Fri, 17 Nov 2023 11:09:02 +0530 Subject: [PATCH 13/16] fix: adding dedicated mappingJson for setOnce --- src/v0/destinations/mp/config.js | 3 + .../mp/data/MPIdentifyConfig.json | 84 ++++++------------- .../destinations/mp/data/MPSetOnceConfig.json | 62 ++++++++++++++ src/v0/destinations/mp/util.js | 3 +- 4 files changed, 92 insertions(+), 60 deletions(-) create mode 100644 src/v0/destinations/mp/data/MPSetOnceConfig.json diff --git a/src/v0/destinations/mp/config.js b/src/v0/destinations/mp/config.js index 41a801e9da..35b40294f5 100644 --- a/src/v0/destinations/mp/config.js +++ b/src/v0/destinations/mp/config.js @@ -11,6 +11,9 @@ const ConfigCategory = { IDENTIFY: { name: 'MPIdentifyConfig', }, + SET_ONCE: { + name: 'MPSetOnceConfig', + }, PROFILE_ANDROID: { name: 'MPProfilePropertiesAndroid', }, diff --git a/src/v0/destinations/mp/data/MPIdentifyConfig.json b/src/v0/destinations/mp/data/MPIdentifyConfig.json index 6934735f67..679ce66d7f 100644 --- a/src/v0/destinations/mp/data/MPIdentifyConfig.json +++ b/src/v0/destinations/mp/data/MPIdentifyConfig.json @@ -1,54 +1,33 @@ [ { "destKey": "$created", - "sourceKeys": ["traits.createdAt", "context.traits.createdAt", "createdAt"], - "required": false + "sourceKeys": "createdAtOnly", + "required": false, + "sourceFromGenericMap": true }, { "destKey": "$email", - "sourceKeys": [ - "traits.email", - "context.traits.email", - "properties.email", - "context.externalId.0.id", - "email" - ], - "required": false + "sourceKeys": "email", + "required": false, + "sourceFromGenericMap": true }, { "destKey": "$first_name", - "sourceKeys": [ - "traits.firstName", - "traits.firstname", - "traits.first_name", - "context.traits.firstName", - "context.traits.firstname", - "context.traits.first_name", - "firstName", - "firstname", - "first_name" - ], - "required": false + "sourceKeys": "firstName", + "required": false, + "sourceFromGenericMap": true }, { "destKey": "$last_name", - "sourceKeys": [ - "traits.lastName", - "traits.lastname", - "traits.last_name", - "context.traits.lastName", - "context.traits.lastname", - "context.traits.last_name", - "lastName", - "lastname", - "last_name" - ], - "required": false + "sourceKeys": "lastName", + "required": false, + "sourceFromGenericMap": true }, { "destKey": "$name", - "sourceKeys": ["traits.name", "context.traits.name", "name"], - "required": false + "sourceKeys": "name", + "required": false, + "sourceFromGenericMap": true }, { "destKey": "$username", @@ -56,20 +35,19 @@ "traits.username", "context.traits.username", "traits.userName", - "context.traits.userName", - "username", - "userName" + "context.traits.userName" ], "required": false }, { "destKey": "$phone", - "sourceKeys": ["traits.phone", "context.traits.phone", "phone"], - "required": false + "sourceKeys": "phone", + "required": false, + "sourceFromGenericMap": true }, { "destKey": "$avatar", - "sourceKeys": ["traits.avatar", "context.traits.avatar", "avatar"], + "sourceKeys": ["traits.avatar", "context.traits.avatar"], "required": false }, { @@ -79,9 +57,7 @@ "context.traits.address.country", "traits.country", "context.traits.country", - "context.location.country", - "country", - "address.country" + "context.location.country" ], "required": false }, @@ -92,9 +68,7 @@ "context.traits.city", "traits.address.city", "context.traits.address.city", - "context.location.city", - "city", - "address.city" + "context.location.city" ], "required": false }, @@ -105,16 +79,13 @@ "context.traits.state", "traits.address.state", "context.traits.address.state", - "context.location.region", - "state", - "address.state", - "location.region" + "context.location.region" ], "required": false }, { "destKey": "$geo_source", - "sourceKeys": ["location.geoSource", "context.location.geoSource"], + "sourceKeys": "context.location.geoSource", "required": false }, { @@ -164,12 +135,7 @@ }, { "destKey": "$unsubscribed", - "sourceKeys": [ - "traits.unsubscribed", - "context.traits.unsubscribed", - "properties.unsubscribed", - "unsubscribed" - ], + "sourceKeys": ["traits.unsubscribed", "context.traits.unsubscribed", "properties.unsubscribed"], "required": false }, { diff --git a/src/v0/destinations/mp/data/MPSetOnceConfig.json b/src/v0/destinations/mp/data/MPSetOnceConfig.json new file mode 100644 index 0000000000..4371827c6c --- /dev/null +++ b/src/v0/destinations/mp/data/MPSetOnceConfig.json @@ -0,0 +1,62 @@ +[ + { + "destKey": "$created", + "sourceKeys": "createdAt", + "required": false + }, + { + "destKey": "$email", + "sourceKeys": "email", + "required": false + }, + { + "destKey": "$first_name", + "sourceKeys": ["firstName", "firstname", "first_name"], + "required": false + }, + { + "destKey": "$last_name", + "sourceKeys": ["lastName", "lastname", "last_name"], + "required": false + }, + { + "destKey": "$name", + "sourceKeys": "name", + "required": false + }, + { + "destKey": "$username", + "sourceKeys": ["username", "userName"], + "required": false + }, + { + "destKey": "$phone", + "sourceKeys": "phone", + "required": false + }, + { + "destKey": "$avatar", + "sourceKeys": "avatar", + "required": false + }, + { + "destKey": "$country_code", + "sourceKeys": ["country", "address.country"], + "required": false + }, + { + "destKey": "$city", + "sourceKeys": ["city", "address.city"], + "required": false + }, + { + "destKey": "$region", + "sourceKeys": ["state", "address.state", "location.region"], + "required": false + }, + { + "destKey": "$unsubscribed", + "sourceKeys": ["unsubscribed"], + "required": false + } +] diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index a2dc2f2563..dd1a719389 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -28,6 +28,7 @@ const { CommonUtils } = require('../../../util/common'); const mPIdentifyConfigJson = mappingConfig[ConfigCategory.IDENTIFY.name]; const mPProfileAndroidConfigJson = mappingConfig[ConfigCategory.PROFILE_ANDROID.name]; const mPProfileIosConfigJson = mappingConfig[ConfigCategory.PROFILE_IOS.name]; +const mPSetOnceConfigJson = mappingConfig[ConfigCategory.SET_ONCE.name]; /** * this function has been used to create @@ -380,7 +381,7 @@ function trimTraits(traits, contextTraits, setOnceProperties) { if (setOnceEligible && Object.keys(setOnceEligible).length > 0) { // Step 2: transform properties eligible as per rudderstack declared identify event mapping // setOnce should have all traits from message.traits and message.context.traits by now - sentOnceTransform = constructPayload(setOnceEligible, mPIdentifyConfigJson); + sentOnceTransform = constructPayload(setOnceEligible, mPSetOnceConfigJson); // Step 3: combine the transformed and custom setOnce traits sentOnceTransform = extractCustomFields( From 51b0313a5d5afa6ddf08e085d551e214e5d6b07d Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Fri, 17 Nov 2023 13:01:53 +0530 Subject: [PATCH 14/16] fix: adding all the fields to the dedicated json --- .../destinations/mp/data/MPSetOnceConfig.json | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/src/v0/destinations/mp/data/MPSetOnceConfig.json b/src/v0/destinations/mp/data/MPSetOnceConfig.json index 4371827c6c..e5aaf851a3 100644 --- a/src/v0/destinations/mp/data/MPSetOnceConfig.json +++ b/src/v0/destinations/mp/data/MPSetOnceConfig.json @@ -56,7 +56,67 @@ }, { "destKey": "$unsubscribed", - "sourceKeys": ["unsubscribed"], + "sourceKeys": "unsubscribed", + "required": false + }, + { + "destKey": "$geo_source", + "sourceKeys": "location.geoSource", + "required": false + }, + { + "destKey": "$timezone", + "sourceKeys": "location.timezone", + "required": false + }, + { + "destKey": "$latitude", + "sourceKeys": "location.latitude", + "required": false + }, + { + "destKey": "$longitude", + "sourceKeys": "location.longitude", + "required": false + }, + { + "destKey": "$carrier", + "sourceKeys": "network.carrier", + "required": false + }, + { + "destKey": "$manufacturer", + "sourceKeys": "device.manufacturer", + "required": false + }, + { + "destKey": "$model", + "sourceKeys": "device.model", + "required": false + }, + { + "destKey": "$screen_height", + "sourceKeys": "screen.height", + "required": false + }, + { + "destKey": "$screen_width", + "sourceKeys": "screen.width", + "required": false + }, + { + "destKey": "$wifi", + "sourceKeys": "network.wifi", + "required": false + }, + { + "destKey": "$initial_referrer", + "sourceKeys": "page.initial_referrer", + "required": false + }, + { + "destKey": "$initial_referring_domain", + "sourceKeys": ["page.initial_referring_domain", "page.initialReferringDomain"], "required": false } ] From 6f9ff79e0326278130fedd49130ec8950ba346d9 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Wed, 22 Nov 2023 15:23:40 +0530 Subject: [PATCH 15/16] fix: addressing review comments --- src/v0/destinations/mp/transform.js | 5 +---- src/v0/destinations/mp/util.js | 11 +++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index cab50b976f..7a7b499637 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -249,10 +249,7 @@ const processIdentifyEvents = async (message, type, destination) => { let setOnceProperties = []; // making payload for set_once properties - if ( - destination.Config.setOnceProperties && - Object.keys(destination.Config.setOnceProperties).length > 0 - ) { + if (destination.Config.setOnceProperties && destination.Config.setOnceProperties.length > 0) { setOnceProperties = parseConfigArray(destination.Config.setOnceProperties, 'property'); seggregatedTraits = trimTraits( messageClone.traits, diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index dd1a719389..d1c646d6ae 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -346,11 +346,10 @@ const parseConfigArray = (arr, key) => { * const setOnceProperties = ['name', 'country', 'address.city']; * * const result = trimTraits(traits, contextTraits, setOnceProperties); - * console.log(result); * // Output: { traits: { age: 30 }, contextTraits: { language: 'English' }, setOnce: { $name: 'John', $country_code: 'USA', city: 'New York'} } */ function trimTraits(traits, contextTraits, setOnceProperties) { - let sentOnceTransform; + let sentOnceTransformedPayload; // Create a copy of the original traits object const traitsCopy = { ...traits }; const contextTraitsCopy = { ...contextTraits }; @@ -381,12 +380,12 @@ function trimTraits(traits, contextTraits, setOnceProperties) { if (setOnceEligible && Object.keys(setOnceEligible).length > 0) { // Step 2: transform properties eligible as per rudderstack declared identify event mapping // setOnce should have all traits from message.traits and message.context.traits by now - sentOnceTransform = constructPayload(setOnceEligible, mPSetOnceConfigJson); + sentOnceTransformedPayload = constructPayload(setOnceEligible, mPSetOnceConfigJson); // Step 3: combine the transformed and custom setOnce traits - sentOnceTransform = extractCustomFields( + sentOnceTransformedPayload = extractCustomFields( setOnceEligible, - sentOnceTransform, + sentOnceTransformedPayload, 'root', MP_IDENTIFY_EXCLUSION_LIST, ); @@ -395,7 +394,7 @@ function trimTraits(traits, contextTraits, setOnceProperties) { return { traits: traitsCopy, contextTraits: contextTraitsCopy, - setOnce: sentOnceTransform || {}, + setOnce: sentOnceTransformedPayload || {}, }; } From 2949fa134bf8336b1e5418a28996e5fded5ab372 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Thu, 23 Nov 2023 17:59:10 +0530 Subject: [PATCH 16/16] feat: review comments addressed --- src/v0/destinations/mp/transform.js | 2 +- src/v0/destinations/mp/util.js | 8 -- src/v0/destinations/mp/util.test.js | 4 +- src/v0/util/index.js | 26 ++++ .../destinations/mp/processor/data.ts | 134 ++++++++++++++++++ 5 files changed, 163 insertions(+), 11 deletions(-) diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 7a7b499637..3d0aaa7c4c 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -17,6 +17,7 @@ const { checkInvalidRtTfEvents, handleRtTfSingleEventError, groupEventsByType, + parseConfigArray, } = require('../../util'); const { ConfigCategory, @@ -35,7 +36,6 @@ const { combineBatchRequestsWithSameJobIds, groupEventsByEndpoint, batchEvents, - parseConfigArray, trimTraits, } = require('./util'); const { CommonUtils } = require('../../../util/common'); diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index d1c646d6ae..bb8f36fdbe 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -325,13 +325,6 @@ const combineBatchRequestsWithSameJobIds = (inputBatches) => { return combineBatches(combineBatches(inputBatches)); }; -const parseConfigArray = (arr, key) => { - if (!arr) { - return []; - } - return arr.map((item) => item[key]); -}; - /** * Trims the traits and contextTraits objects based on the setOnceProperties array and returns an object containing the modified traits, contextTraits, and setOnce properties. * @@ -406,6 +399,5 @@ module.exports = { generateBatchedPayloadForArray, batchEvents, combineBatchRequestsWithSameJobIds, - parseConfigArray, trimTraits, }; diff --git a/src/v0/destinations/mp/util.test.js b/src/v0/destinations/mp/util.test.js index 028dfa3be3..fbaa6f9b9f 100644 --- a/src/v0/destinations/mp/util.test.js +++ b/src/v0/destinations/mp/util.test.js @@ -683,14 +683,14 @@ describe('Mixpanel utils test', () => { }); it('should add properties to the setOnce property when given setOnceProperties array with existent nested properties', () => { - const traits = { name: 'John', age: 30, address: { city: 'kolkata' } }; + const traits = { name: 'John', age: 30, address: { city: 'kolkata' }, isAdult: false }; const contextTraits = { email: 'john@example.com' }; const setOnceProperties = ['name', 'email', 'address.city']; const result = trimTraits(traits, contextTraits, setOnceProperties); expect(result).toEqual({ - traits: { age: 30, address: {} }, + traits: { age: 30, address: {}, isAdult: false }, contextTraits: {}, setOnce: { $name: 'John', $email: 'john@example.com', $city: 'kolkata' }, }); diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 0296895662..fee1d7a96d 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -2083,6 +2083,31 @@ const IsGzipSupported = (reqMetadata = {}) => { return false; }; +/** + * Returns an array containing the values of the specified key from each object in the input array. + * If the input array is falsy (null, undefined, empty array), an empty array is returned. + * + * @param {Array} arr - The input array from which values will be extracted. + * @param {string} key - The key of the property whose values will be extracted from each object in the input array. + * @returns {Array} - A new array containing the values of the specified key from each object in the input array. + * + * @example + * const configArray = [ + * { name: 'John', age: 25 }, + * { name: 'Jane', age: 30 }, + * { name: 'Bob', age: 35 } + * ]; + * + * const result = parseConfigArray(configArray, 'name'); + * Output: ['John', 'Jane', 'Bob'] + */ +const parseConfigArray = (arr, key) => { + if (!arr) { + return []; + } + return arr.map((item) => item[key]); +}; + // ======================================================================== // EXPORTS // ======================================================================== @@ -2192,4 +2217,5 @@ module.exports = { isValidInteger, isNewStatusCodesAccepted, IsGzipSupported, + parseConfigArray, }; diff --git a/test/integrations/destinations/mp/processor/data.ts b/test/integrations/destinations/mp/processor/data.ts index 30da96ff9f..a6ba51ee78 100644 --- a/test/integrations/destinations/mp/processor/data.ts +++ b/test/integrations/destinations/mp/processor/data.ts @@ -5987,4 +5987,138 @@ export const data = [ }, }, }, + { + name: 'mp', + description: 'Test Set Once Property with anonymousId', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + description: 'Alias: with property beyond and within exclusion list', + destination: destinationWithSetOnceProperty, + message: { + anonymousId: 'dummyAnnonymousId', + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.5', + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.5', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + traits: { + address: { + city: 'Disney', + }, + country: 'USA', + email: 'TestSanity@disney.com', + firstName: 'Mickey test', + lastName: 'VarChange', + createdAt: '2020-01-23T08:54:02.362Z', + nationality: 'USA', + random: 'superProp', + }, + page: { + path: '/destinations/mixpanel', + referrer: '', + search: '', + title: '', + url: 'https://docs.rudderstack.com/destinations/mixpanel', + category: 'destination', + initial_referrer: 'https://docs.rudderstack.com', + initial_referring_domain: 'docs.rudderstack.com', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', + }, + integrations: { + All: true, + }, + page: { + path: '/destinations/mixpanel', + referrer: '', + search: '', + title: '', + url: 'https://docs.rudderstack.com/destinations/mixpanel', + category: 'destination', + initial_referrer: 'https://docs.rudderstack.com', + initial_referring_domain: 'docs.rudderstack.com', + }, + request_ip: '[::1]:53709', + type: 'identify', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.mixpanel.com/engage/', + headers: {}, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: { + batch: + '[{"$set_once":{"$first_name":"Mickey test","$city":"Disney","nationality":"USA"},"$token":"dummyToken","$distinct_id":"$device:dummyAnnonymousId"}]', + }, + XML: {}, + FORM: {}, + }, + files: {}, + userId: 'dummyAnnonymousId', + }, + statusCode: 200, + }, + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.mixpanel.com/engage/', + headers: {}, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: { + batch: + '[{"$set":{"$created":"2020-01-23T08:54:02.362Z","$email":"TestSanity@disney.com","$country_code":"USA","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","random":"superProp","$lastName":"VarChange","$browser":"Chrome","$browser_version":"79.0.3945.117"},"$token":"dummyToken","$distinct_id":"$device:dummyAnnonymousId","$ip":"0.0.0.0","$time":null}]', + }, + XML: {}, + FORM: {}, + }, + files: {}, + userId: 'dummyAnnonymousId', + }, + statusCode: 200, + }, + ], + }, + }, + }, ];